From 233a90c0a30b597aace585f0f7fc6b61f9452e19 Mon Sep 17 00:00:00 2001 From: ary1992 Date: Thu, 3 Aug 2023 12:17:33 +0530 Subject: [PATCH] Use `old` restmapper - Latest restmapper implementation doesnot handle cases where an API Group / GroupVersion is not present in the system. In such cases Clients expect No{Kind,Resource}MatchErrors. - Upstream fix is already open https://github.com/kubernetes-sigs/controller-runtime/pull/2425. - Revert this commit once the fix is available. --- cmd/gardener-apiserver/.import-restrictions | 2 +- cmd/gardener-resource-manager/app/app.go | 8 +- cmd/gardenlet/app/app.go | 20 ++ extensions/.import-restrictions | 1 + extensions/pkg/util/shoot_clients.go | 9 +- go.mod | 3 +- pkg/client/kubernetes/runtime_client.go | 12 +- skaffold-operator.yaml | 2 + skaffold.yaml | 7 + .../gardenlet/seed/care/care_suite_test.go | 6 + .../gardenlet/seed/seed/seed_test.go | 7 +- .../operator/garden/care/care_suite_test.go | 6 +- .../operator/garden/garden/garden_test.go | 7 +- .../health/health_suite_test.go | 6 + .../pkg/apiutil/dynamicrestmapper.go | 301 ++++++++++++++++++ .../pkg/apiutil/lazyrestmapper.go | 266 ++++++++++++++++ 16 files changed, 629 insertions(+), 34 deletions(-) create mode 100644 third_party/controller-runtime/pkg/apiutil/dynamicrestmapper.go create mode 100644 third_party/controller-runtime/pkg/apiutil/lazyrestmapper.go diff --git a/cmd/gardener-apiserver/.import-restrictions b/cmd/gardener-apiserver/.import-restrictions index bd23f32fcad..7566736a049 100644 --- a/cmd/gardener-apiserver/.import-restrictions +++ b/cmd/gardener-apiserver/.import-restrictions @@ -6,4 +6,4 @@ rules: # make sure apiserver can build without these packages - github.com/gardener/gardener/charts - github.com/gardener/gardener/pkg/operation - - github.com/gardener/gardener/third_party + - github.com/gardener/gardener/third_party/apiserver \ No newline at end of file diff --git a/cmd/gardener-resource-manager/app/app.go b/cmd/gardener-resource-manager/app/app.go index 754dc5b1a97..c1ab06ff800 100644 --- a/cmd/gardener-resource-manager/app/app.go +++ b/cmd/gardener-resource-manager/app/app.go @@ -27,6 +27,7 @@ import ( "github.com/go-logr/logr" "github.com/spf13/cobra" "github.com/spf13/pflag" + "golang.org/x/time/rate" corev1 "k8s.io/api/core/v1" eventsv1 "k8s.io/api/events/v1" eventsv1beta1 "k8s.io/api/events/v1beta1" @@ -38,7 +39,6 @@ import ( "k8s.io/klog/v2" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/cluster" controllerconfig "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -56,6 +56,7 @@ import ( resourcemanagerclient "github.com/gardener/gardener/pkg/resourcemanager/client" "github.com/gardener/gardener/pkg/resourcemanager/controller" "github.com/gardener/gardener/pkg/resourcemanager/webhook" + thirdpartyapiutil "github.com/gardener/gardener/third_party/controller-runtime/pkg/apiutil" ) // Name is a const for the name of this component. @@ -206,9 +207,10 @@ func run(ctx context.Context, log logr.Logger, cfg *config.ResourceManagerConfig // use dynamic rest mapper for target cluster, which will automatically rediscover resources on NoMatchErrors // but is rate-limited to not issue to many discovery calls (rate-limit shared across all reconciliations) opts.MapperProvider = func(config *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) { - return apiutil.NewDynamicRESTMapper( + return thirdpartyapiutil.NewDynamicRESTMapper( config, - httpClient, + thirdpartyapiutil.WithLazyDiscovery, + thirdpartyapiutil.WithLimiter(rate.NewLimiter(rate.Every(1*time.Minute), 1)), // rediscover at maximum every minute ) } diff --git a/cmd/gardenlet/app/app.go b/cmd/gardenlet/app/app.go index 3f3ba1005ca..cd13065859d 100644 --- a/cmd/gardenlet/app/app.go +++ b/cmd/gardenlet/app/app.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "net" + "net/http" "os" goruntime "runtime" "strconv" @@ -26,11 +27,13 @@ import ( "github.com/go-logr/logr" "github.com/spf13/cobra" "github.com/spf13/pflag" + "golang.org/x/time/rate" certificatesv1 "k8s.io/api/certificates/v1" coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" eventsv1 "k8s.io/api/events/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/wait" @@ -72,6 +75,7 @@ import ( "github.com/gardener/gardener/pkg/utils/flow" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" + thirdpartyapiutil "github.com/gardener/gardener/third_party/controller-runtime/pkg/apiutil" ) // Name is a const for the name of this component. @@ -162,6 +166,14 @@ func run(ctx context.Context, cancel context.CancelFunc, log logr.Logger, cfg *c RecoverPanic: pointer.Bool(true), }, + MapperProvider: func(config *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) { + return thirdpartyapiutil.NewDynamicRESTMapper( + config, + thirdpartyapiutil.WithLazyDiscovery, + thirdpartyapiutil.WithLimiter(rate.NewLimiter(rate.Every(1*time.Minute), 1)), // rediscover at maximum every minute + ) + }, + Client: client.Options{ Cache: &client.CacheOptions{ DisableFor: []client.Object{ @@ -330,6 +342,14 @@ func (g *garden) Start(ctx context.Context) error { Reader: uncachedClient, }, nil } + + opts.MapperProvider = func(config *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) { + return thirdpartyapiutil.NewDynamicRESTMapper( + config, + thirdpartyapiutil.WithLazyDiscovery, + thirdpartyapiutil.WithLimiter(rate.NewLimiter(rate.Every(1*time.Minute), 1)), // rediscover at maximum every minute + ) + } }) if err != nil { return fmt.Errorf("failed creating garden cluster object: %w", err) diff --git a/extensions/.import-restrictions b/extensions/.import-restrictions index 76e47d9c3e8..7f5b623d28e 100644 --- a/extensions/.import-restrictions +++ b/extensions/.import-restrictions @@ -24,3 +24,4 @@ rules: - github.com/gardener/gardener/pkg/mock - github.com/gardener/gardener/pkg/resourcemanager/controller/garbagecollector/references - github.com/gardener/gardener/pkg/utils + - github.com/gardener/gardener/third_party/controller-runtime/pkg/apiutil diff --git a/extensions/pkg/util/shoot_clients.go b/extensions/pkg/util/shoot_clients.go index 98d71308832..b5fe2e56ffd 100644 --- a/extensions/pkg/util/shoot_clients.go +++ b/extensions/pkg/util/shoot_clients.go @@ -26,7 +26,6 @@ import ( "k8s.io/client-go/rest" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" extensionsconfig "github.com/gardener/gardener/extensions/pkg/apis/config" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" @@ -34,6 +33,7 @@ import ( kubernetesclient "github.com/gardener/gardener/pkg/client/kubernetes" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/gardener/gardener/pkg/utils/secrets" + thirdpartyapiutil "github.com/gardener/gardener/third_party/controller-runtime/pkg/apiutil" ) // ShootClients bundles together several clients for the shoot cluster. @@ -108,12 +108,7 @@ func NewClientForShoot(ctx context.Context, c client.Client, namespace string, o ApplyRESTOptions(shootRESTConfig, restOptions) if opts.Mapper == nil { - httpClient, err := rest.HTTPClientFor(shootRESTConfig) - if err != nil { - return nil, nil, fmt.Errorf("failed to get HTTP client for config: %w", err) - } - - mapper, err := apiutil.NewDynamicRESTMapper(shootRESTConfig, httpClient) + mapper, err := thirdpartyapiutil.NewDynamicRESTMapper(shootRESTConfig) if err != nil { return nil, nil, fmt.Errorf("failed to create new DynamicRESTMapper: %w", err) } diff --git a/go.mod b/go.mod index 0f7eab855aa..3d8aba7897e 100644 --- a/go.mod +++ b/go.mod @@ -64,11 +64,10 @@ require ( sigs.k8s.io/controller-runtime v0.15.0 sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20221212190805-d4f1e822ca11 // v0.14.1 sigs.k8s.io/controller-tools v0.11.3 + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 sigs.k8s.io/yaml v1.3.0 ) -require sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - require ( github.com/BurntSushi/toml v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect diff --git a/pkg/client/kubernetes/runtime_client.go b/pkg/client/kubernetes/runtime_client.go index 212a80730bc..0bf30015f3b 100644 --- a/pkg/client/kubernetes/runtime_client.go +++ b/pkg/client/kubernetes/runtime_client.go @@ -19,6 +19,7 @@ import ( "time" "github.com/go-logr/logr" + "golang.org/x/time/rate" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" @@ -29,6 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/apiutil" kubernetescache "github.com/gardener/gardener/pkg/client/kubernetes/cache" + thirdpartyapiutil "github.com/gardener/gardener/third_party/controller-runtime/pkg/apiutil" ) const ( @@ -55,15 +57,11 @@ func setCacheOptionsDefaults(options *cache.Options) error { func setClientOptionsDefaults(config *rest.Config, options *client.Options) error { if options.Mapper == nil { - httpClient, err := rest.HTTPClientFor(config) - if err != nil { - return fmt.Errorf("failed to get HTTP client for config: %w", err) - } - // default the client's REST mapper to a dynamic REST mapper (automatically rediscovers resources on NoMatchErrors) - mapper, err := apiutil.NewDynamicRESTMapper( + mapper, err := thirdpartyapiutil.NewDynamicRESTMapper( config, - httpClient, + thirdpartyapiutil.WithLazyDiscovery, + thirdpartyapiutil.WithLimiter(rate.NewLimiter(rate.Every(5*time.Second), 1)), ) if err != nil { return fmt.Errorf("failed to create new DynamicRESTMapper: %w", err) diff --git a/skaffold-operator.yaml b/skaffold-operator.yaml index cdc41a52401..b76ed13da25 100644 --- a/skaffold-operator.yaml +++ b/skaffold-operator.yaml @@ -211,6 +211,7 @@ build: - pkg/utils/version - plugin/pkg - third_party/gopkg.in/yaml.v2 + - third_party/controller-runtime/pkg/apiutil - vendor - VERSION ldflags: @@ -312,6 +313,7 @@ build: - pkg/utils/validation/features - pkg/utils/validation/kubernetesversion - pkg/utils/version + - third_party/controller-runtime/pkg/apiutil - vendor - VERSION ldflags: diff --git a/skaffold.yaml b/skaffold.yaml index b6d0dff9202..a3a57d4ba05 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -201,6 +201,7 @@ build: - plugin/pkg/shoot/validator - plugin/pkg/shoot/vpa - plugin/pkg/utils + - third_party/controller-runtime/pkg/apiutil - vendor - VERSION ldflags: @@ -306,6 +307,7 @@ build: - pkg/utils/validation/kubernetesversion - pkg/utils/version - third_party/gopkg.in/yaml.v2 + - third_party/controller-runtime/pkg/apiutil - vendor - VERSION ldflags: @@ -369,6 +371,7 @@ build: - pkg/utils/validation/cidr - pkg/utils/validation/kubernetesversion - pkg/utils/version + - third_party/controller-runtime/pkg/apiutil - vendor - VERSION ldflags: @@ -446,6 +449,7 @@ build: - pkg/utils/version - third_party/apiserver/pkg/apis/audit/v1alpha1 - third_party/apiserver/pkg/apis/audit/v1beta1 + - third_party/controller-runtime/pkg/apiutil - vendor - VERSION ldflags: @@ -672,6 +676,7 @@ build: - pkg/utils/validation/kubernetesversion - pkg/utils/version - third_party/gopkg.in/yaml.v2 + - third_party/controller-runtime/pkg/apiutil - vendor - VERSION ldflags: @@ -980,6 +985,7 @@ build: - pkg/utils/validation/kubernetesversion - pkg/utils/version - third_party/gopkg.in/yaml.v2 + - third_party/controller-runtime/pkg/apiutil - vendor - VERSION ldflags: @@ -1081,6 +1087,7 @@ build: - pkg/utils/validation/features - pkg/utils/validation/kubernetesversion - pkg/utils/version + - third_party/controller-runtime/pkg/apiutil - vendor - VERSION ldflags: diff --git a/test/integration/gardenlet/seed/care/care_suite_test.go b/test/integration/gardenlet/seed/care/care_suite_test.go index 4a1c74d8102..456e1d1249f 100644 --- a/test/integration/gardenlet/seed/care/care_suite_test.go +++ b/test/integration/gardenlet/seed/care/care_suite_test.go @@ -16,6 +16,7 @@ package care_test import ( "context" + "net/http" "path/filepath" "testing" "time" @@ -24,6 +25,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -44,6 +46,7 @@ import ( "github.com/gardener/gardener/pkg/gardenlet/features" "github.com/gardener/gardener/pkg/logger" . "github.com/gardener/gardener/pkg/utils/test/matchers" + thirdpartyapiutil "github.com/gardener/gardener/third_party/controller-runtime/pkg/apiutil" ) func TestCare(t *testing.T) { @@ -135,6 +138,9 @@ var _ = BeforeSuite(func() { }, }, }, + MapperProvider: func(config *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) { + return thirdpartyapiutil.NewDynamicRESTMapper(config) + }, }) Expect(err).NotTo(HaveOccurred()) mgrClient = mgr.GetClient() diff --git a/test/integration/gardenlet/seed/seed/seed_test.go b/test/integration/gardenlet/seed/seed/seed_test.go index 07ee64e66d6..508d1326705 100644 --- a/test/integration/gardenlet/seed/seed/seed_test.go +++ b/test/integration/gardenlet/seed/seed/seed_test.go @@ -28,11 +28,9 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/rest" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/gardener/gardener/pkg/api/indexer" @@ -58,6 +56,7 @@ import ( "github.com/gardener/gardener/pkg/utils/test" . "github.com/gardener/gardener/pkg/utils/test/matchers" "github.com/gardener/gardener/test/utils/namespacefinalizer" + thirdpartyapiutil "github.com/gardener/gardener/third_party/controller-runtime/pkg/apiutil" ) var _ = Describe("Seed controller tests", func() { @@ -88,9 +87,7 @@ var _ = Describe("Seed controller tests", func() { }) By("Setup manager") - httpClient, err := rest.HTTPClientFor(restConfig) - Expect(err).NotTo(HaveOccurred()) - mapper, err := apiutil.NewDynamicRESTMapper(restConfig, httpClient) + mapper, err := thirdpartyapiutil.NewDynamicRESTMapper(restConfig) Expect(err).NotTo(HaveOccurred()) mgr, err := manager.New(restConfig, manager.Options{ diff --git a/test/integration/operator/garden/care/care_suite_test.go b/test/integration/operator/garden/care/care_suite_test.go index ba29b612d06..d7bb5f2ad12 100644 --- a/test/integration/operator/garden/care/care_suite_test.go +++ b/test/integration/operator/garden/care/care_suite_test.go @@ -29,7 +29,6 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -45,6 +44,7 @@ import ( "github.com/gardener/gardener/pkg/operator/controller/garden/care" "github.com/gardener/gardener/pkg/operator/features" . "github.com/gardener/gardener/pkg/utils/test/matchers" + thirdpartyapiutil "github.com/gardener/gardener/third_party/controller-runtime/pkg/apiutil" ) func TestGarden(t *testing.T) { @@ -127,9 +127,7 @@ var _ = BeforeSuite(func() { }) By("Setup manager") - httpClient, err := rest.HTTPClientFor(restConfig) - Expect(err).NotTo(HaveOccurred()) - mapper, err := apiutil.NewDynamicRESTMapper(restConfig, httpClient) + mapper, err := thirdpartyapiutil.NewDynamicRESTMapper(restConfig) Expect(err).NotTo(HaveOccurred()) mgr, err := manager.New(restConfig, manager.Options{ diff --git a/test/integration/operator/garden/garden/garden_test.go b/test/integration/operator/garden/garden/garden_test.go index 31176cc3458..18c6ebff3e4 100644 --- a/test/integration/operator/garden/garden/garden_test.go +++ b/test/integration/operator/garden/garden/garden_test.go @@ -29,13 +29,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/rest" clientcmdlatest "k8s.io/client-go/tools/clientcmd/api/latest" clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/manager" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" @@ -65,6 +63,7 @@ import ( "github.com/gardener/gardener/pkg/utils/test" . "github.com/gardener/gardener/pkg/utils/test/matchers" "github.com/gardener/gardener/test/utils/operationannotation" + thirdpartyapiutil "github.com/gardener/gardener/third_party/controller-runtime/pkg/apiutil" ) var _ = Describe("Garden controller tests", func() { @@ -116,9 +115,7 @@ var _ = Describe("Garden controller tests", func() { }) By("Setup manager") - httpClient, err := rest.HTTPClientFor(restConfig) - Expect(err).NotTo(HaveOccurred()) - mapper, err := apiutil.NewDynamicRESTMapper(restConfig, httpClient) + mapper, err := thirdpartyapiutil.NewDynamicRESTMapper(restConfig) Expect(err).NotTo(HaveOccurred()) mgr, err := manager.New(restConfig, manager.Options{ diff --git a/test/integration/resourcemanager/health/health_suite_test.go b/test/integration/resourcemanager/health/health_suite_test.go index fdfac979151..fcc21f1b96d 100644 --- a/test/integration/resourcemanager/health/health_suite_test.go +++ b/test/integration/resourcemanager/health/health_suite_test.go @@ -16,6 +16,7 @@ package health_test import ( "context" + "net/http" "path/filepath" "testing" "time" @@ -24,6 +25,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" "k8s.io/utils/pointer" @@ -40,6 +42,7 @@ import ( "github.com/gardener/gardener/pkg/resourcemanager/controller/health/progressing" resourcemanagerpredicate "github.com/gardener/gardener/pkg/resourcemanager/predicate" . "github.com/gardener/gardener/pkg/utils/test/matchers" + thirdpartyapiutil "github.com/gardener/gardener/third_party/controller-runtime/pkg/apiutil" ) func TestHealth(t *testing.T) { @@ -107,6 +110,9 @@ var _ = BeforeSuite(func() { Scheme: resourcemanagerclient.CombinedScheme, MetricsBindAddress: "0", Namespace: testNamespace.Name, + MapperProvider: func(config *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) { + return thirdpartyapiutil.NewDynamicRESTMapper(config) + }, }) Expect(err).NotTo(HaveOccurred()) diff --git a/third_party/controller-runtime/pkg/apiutil/dynamicrestmapper.go b/third_party/controller-runtime/pkg/apiutil/dynamicrestmapper.go new file mode 100644 index 00000000000..6b9dcf68adf --- /dev/null +++ b/third_party/controller-runtime/pkg/apiutil/dynamicrestmapper.go @@ -0,0 +1,301 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiutil + +import ( + "sync" + "sync/atomic" + + "golang.org/x/time/rate" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" +) + +// dynamicRESTMapper is a RESTMapper that dynamically discovers resource +// types at runtime. +type dynamicRESTMapper struct { + mu sync.RWMutex // protects the following fields + staticMapper meta.RESTMapper + limiter *rate.Limiter + newMapper func() (meta.RESTMapper, error) + + lazy bool + // Used for lazy init. + inited uint32 + initMtx sync.Mutex + + useLazyRestmapper bool +} + +// DynamicRESTMapperOption is a functional option on the dynamicRESTMapper. +type DynamicRESTMapperOption func(*dynamicRESTMapper) error + +// WithLimiter sets the RESTMapper's underlying limiter to lim. +func WithLimiter(lim *rate.Limiter) DynamicRESTMapperOption { + return func(drm *dynamicRESTMapper) error { + drm.limiter = lim + return nil + } +} + +// WithLazyDiscovery prevents the RESTMapper from discovering REST mappings +// until an API call is made. +var WithLazyDiscovery DynamicRESTMapperOption = func(drm *dynamicRESTMapper) error { + drm.lazy = true + return nil +} + +// WithExperimentalLazyMapper enables experimental more advanced Lazy Restmapping mechanism. +var WithExperimentalLazyMapper DynamicRESTMapperOption = func(drm *dynamicRESTMapper) error { + drm.useLazyRestmapper = true + return nil +} + +// WithCustomMapper supports setting a custom RESTMapper refresher instead of +// the default method, which uses a discovery client. +// +// This exists mainly for testing, but can be useful if you need tighter control +// over how discovery is performed, which discovery endpoints are queried, etc. +func WithCustomMapper(newMapper func() (meta.RESTMapper, error)) DynamicRESTMapperOption { + return func(drm *dynamicRESTMapper) error { + drm.newMapper = newMapper + return nil + } +} + +// NewDynamicRESTMapper returns a dynamic RESTMapper for cfg. The dynamic +// RESTMapper dynamically discovers resource types at runtime. opts +// configure the RESTMapper. +func NewDynamicRESTMapper(cfg *rest.Config, opts ...DynamicRESTMapperOption) (meta.RESTMapper, error) { + client, err := discovery.NewDiscoveryClientForConfig(cfg) + if err != nil { + return nil, err + } + drm := &dynamicRESTMapper{ + limiter: rate.NewLimiter(rate.Limit(defaultRefillRate), defaultLimitSize), + newMapper: func() (meta.RESTMapper, error) { + groupResources, err := restmapper.GetAPIGroupResources(client) + if err != nil { + return nil, err + } + return restmapper.NewDiscoveryRESTMapper(groupResources), nil + }, + } + for _, opt := range opts { + if err = opt(drm); err != nil { + return nil, err + } + } + if drm.useLazyRestmapper { + return newLazyRESTMapperWithClient(client) + } + if !drm.lazy { + if err := drm.setStaticMapper(); err != nil { + return nil, err + } + } + return drm, nil +} + +var ( + // defaultRefilRate is the default rate at which potential calls are + // added back to the "bucket" of allowed calls. + defaultRefillRate = 5 + // defaultLimitSize is the default starting/max number of potential calls + // per second. Once a call is used, it's added back to the bucket at a rate + // of defaultRefillRate per second. + defaultLimitSize = 5 +) + +// setStaticMapper sets drm's staticMapper by querying its client, regardless +// of reload backoff. +func (drm *dynamicRESTMapper) setStaticMapper() error { + newMapper, err := drm.newMapper() + if err != nil { + return err + } + drm.staticMapper = newMapper + return nil +} + +// init initializes drm only once if drm is lazy. +func (drm *dynamicRESTMapper) init() (err error) { + // skip init if drm is not lazy or has initialized + if !drm.lazy || atomic.LoadUint32(&drm.inited) != 0 { + return nil + } + + drm.initMtx.Lock() + defer drm.initMtx.Unlock() + if drm.inited == 0 { + if err = drm.setStaticMapper(); err == nil { + atomic.StoreUint32(&drm.inited, 1) + } + } + return err +} + +// checkAndReload attempts to call the given callback, which is assumed to be dependent +// on the data in the restmapper. +// +// If the callback returns an error matching meta.IsNoMatchErr, it will attempt to reload +// the RESTMapper's data and re-call the callback once that's occurred. +// If the callback returns any other error, the function will return immediately regardless. +// +// It will take care of ensuring that reloads are rate-limited and that extraneous calls +// aren't made. If a reload would exceed the limiters rate, it returns the error return by +// the callback. +// It's thread-safe, and worries about thread-safety for the callback (so the callback does +// not need to attempt to lock the restmapper). +func (drm *dynamicRESTMapper) checkAndReload(checkNeedsReload func() error) error { + // first, check the common path -- data is fresh enough + // (use an IIFE for the lock's defer) + err := func() error { + drm.mu.RLock() + defer drm.mu.RUnlock() + + return checkNeedsReload() + }() + + needsReload := meta.IsNoMatchError(err) + if !needsReload { + return err + } + + // if the data wasn't fresh, we'll need to try and update it, so grab the lock... + drm.mu.Lock() + defer drm.mu.Unlock() + + // ... and double-check that we didn't reload in the meantime + err = checkNeedsReload() + needsReload = meta.IsNoMatchError(err) + if !needsReload { + return err + } + + // we're still stale, so grab a rate-limit token if we can... + if !drm.limiter.Allow() { + // return error from static mapper here, we have refreshed often enough (exceeding rate of provided limiter) + // so that client's can handle this the same way as a "normal" NoResourceMatchError / NoKindMatchError + return err + } + + // ...reload... + if err := drm.setStaticMapper(); err != nil { + return err + } + + // ...and return the results of the closure regardless + return checkNeedsReload() +} + +// TODO: wrap reload errors on NoKindMatchError with go 1.13 errors. + +func (drm *dynamicRESTMapper) KindFor(resource schema.GroupVersionResource) (schema.GroupVersionKind, error) { + if err := drm.init(); err != nil { + return schema.GroupVersionKind{}, err + } + var gvk schema.GroupVersionKind + err := drm.checkAndReload(func() error { + var err error + gvk, err = drm.staticMapper.KindFor(resource) + return err + }) + return gvk, err +} + +func (drm *dynamicRESTMapper) KindsFor(resource schema.GroupVersionResource) ([]schema.GroupVersionKind, error) { + if err := drm.init(); err != nil { + return nil, err + } + var gvks []schema.GroupVersionKind + err := drm.checkAndReload(func() error { + var err error + gvks, err = drm.staticMapper.KindsFor(resource) + return err + }) + return gvks, err +} + +func (drm *dynamicRESTMapper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, error) { + if err := drm.init(); err != nil { + return schema.GroupVersionResource{}, err + } + + var gvr schema.GroupVersionResource + err := drm.checkAndReload(func() error { + var err error + gvr, err = drm.staticMapper.ResourceFor(input) + return err + }) + return gvr, err +} + +func (drm *dynamicRESTMapper) ResourcesFor(input schema.GroupVersionResource) ([]schema.GroupVersionResource, error) { + if err := drm.init(); err != nil { + return nil, err + } + var gvrs []schema.GroupVersionResource + err := drm.checkAndReload(func() error { + var err error + gvrs, err = drm.staticMapper.ResourcesFor(input) + return err + }) + return gvrs, err +} + +func (drm *dynamicRESTMapper) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) { + if err := drm.init(); err != nil { + return nil, err + } + var mapping *meta.RESTMapping + err := drm.checkAndReload(func() error { + var err error + mapping, err = drm.staticMapper.RESTMapping(gk, versions...) + return err + }) + return mapping, err +} + +func (drm *dynamicRESTMapper) RESTMappings(gk schema.GroupKind, versions ...string) ([]*meta.RESTMapping, error) { + if err := drm.init(); err != nil { + return nil, err + } + var mappings []*meta.RESTMapping + err := drm.checkAndReload(func() error { + var err error + mappings, err = drm.staticMapper.RESTMappings(gk, versions...) + return err + }) + return mappings, err +} + +func (drm *dynamicRESTMapper) ResourceSingularizer(resource string) (string, error) { + if err := drm.init(); err != nil { + return "", err + } + var singular string + err := drm.checkAndReload(func() error { + var err error + singular, err = drm.staticMapper.ResourceSingularizer(resource) + return err + }) + return singular, err +} diff --git a/third_party/controller-runtime/pkg/apiutil/lazyrestmapper.go b/third_party/controller-runtime/pkg/apiutil/lazyrestmapper.go new file mode 100644 index 00000000000..e9b1e710c2f --- /dev/null +++ b/third_party/controller-runtime/pkg/apiutil/lazyrestmapper.go @@ -0,0 +1,266 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiutil + +import ( + "fmt" + "sync" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/restmapper" +) + +// lazyRESTMapper is a RESTMapper that will lazily query the provided +// client for discovery information to do REST mappings. +type lazyRESTMapper struct { + mapper meta.RESTMapper + client *discovery.DiscoveryClient + knownGroups map[string]*restmapper.APIGroupResources + apiGroups []metav1.APIGroup + + // mutex to provide thread-safe mapper reloading. + mu sync.Mutex +} + +// newLazyRESTMapperWithClient initializes a LazyRESTMapper with a custom discovery client. +func newLazyRESTMapperWithClient(discoveryClient *discovery.DiscoveryClient) (meta.RESTMapper, error) { + return &lazyRESTMapper{ + mapper: restmapper.NewDiscoveryRESTMapper([]*restmapper.APIGroupResources{}), + client: discoveryClient, + knownGroups: map[string]*restmapper.APIGroupResources{}, + apiGroups: []metav1.APIGroup{}, + }, nil +} + +// KindFor implements Mapper.KindFor. +func (m *lazyRESTMapper) KindFor(resource schema.GroupVersionResource) (schema.GroupVersionKind, error) { + res, err := m.mapper.KindFor(resource) + if meta.IsNoMatchError(err) { + if err = m.addKnownGroupAndReload(resource.Group, resource.Version); err != nil { + return res, err + } + + res, err = m.mapper.KindFor(resource) + } + + return res, err +} + +// KindsFor implements Mapper.KindsFor. +func (m *lazyRESTMapper) KindsFor(resource schema.GroupVersionResource) ([]schema.GroupVersionKind, error) { + res, err := m.mapper.KindsFor(resource) + if meta.IsNoMatchError(err) { + if err = m.addKnownGroupAndReload(resource.Group, resource.Version); err != nil { + return res, err + } + + res, err = m.mapper.KindsFor(resource) + } + + return res, err +} + +// ResourceFor implements Mapper.ResourceFor. +func (m *lazyRESTMapper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, error) { + res, err := m.mapper.ResourceFor(input) + if meta.IsNoMatchError(err) { + if err = m.addKnownGroupAndReload(input.Group, input.Version); err != nil { + return res, err + } + + res, err = m.mapper.ResourceFor(input) + } + + return res, err +} + +// ResourcesFor implements Mapper.ResourcesFor. +func (m *lazyRESTMapper) ResourcesFor(input schema.GroupVersionResource) ([]schema.GroupVersionResource, error) { + res, err := m.mapper.ResourcesFor(input) + if meta.IsNoMatchError(err) { + if err = m.addKnownGroupAndReload(input.Group, input.Version); err != nil { + return res, err + } + + res, err = m.mapper.ResourcesFor(input) + } + + return res, err +} + +// RESTMapping implements Mapper.RESTMapping. +func (m *lazyRESTMapper) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) { + res, err := m.mapper.RESTMapping(gk, versions...) + if meta.IsNoMatchError(err) { + if err = m.addKnownGroupAndReload(gk.Group, versions...); err != nil { + return res, err + } + + res, err = m.mapper.RESTMapping(gk, versions...) + } + + return res, err +} + +// RESTMappings implements Mapper.RESTMappings. +func (m *lazyRESTMapper) RESTMappings(gk schema.GroupKind, versions ...string) ([]*meta.RESTMapping, error) { + res, err := m.mapper.RESTMappings(gk, versions...) + if meta.IsNoMatchError(err) { + if err = m.addKnownGroupAndReload(gk.Group, versions...); err != nil { + return res, err + } + + res, err = m.mapper.RESTMappings(gk, versions...) + } + + return res, err +} + +// ResourceSingularizer implements Mapper.ResourceSingularizer. +func (m *lazyRESTMapper) ResourceSingularizer(resource string) (string, error) { + return m.mapper.ResourceSingularizer(resource) +} + +// addKnownGroupAndReload reloads the mapper with updated information about missing API group. +// versions can be specified for partial updates, for instance for v1beta1 version only. +func (m *lazyRESTMapper) addKnownGroupAndReload(groupName string, versions ...string) error { + m.mu.Lock() + defer m.mu.Unlock() + + // If no specific versions are set by user, we will scan all available ones for the API group. + // This operation requires 2 requests: /api and /apis, but only once. For all subsequent calls + // this data will be taken from cache. + if len(versions) == 0 { + apiGroup, err := m.findAPIGroupByNameLocked(groupName) + if err != nil { + return err + } + for _, version := range apiGroup.Versions { + versions = append(versions, version.Version) + } + } + + // Create or fetch group resources from cache. + groupResources := &restmapper.APIGroupResources{ + Group: metav1.APIGroup{Name: groupName}, + VersionedResources: make(map[string][]metav1.APIResource), + } + if _, ok := m.knownGroups[groupName]; ok { + groupResources = m.knownGroups[groupName] + } + + // Update information for group resources about versioned resources. + // The number of API calls is equal to the number of versions: /apis//. + groupVersionResources, err := m.fetchGroupVersionResources(groupName, versions...) + if err != nil { + return fmt.Errorf("failed to get API group resources: %w", err) + } + for version, resources := range groupVersionResources { + groupResources.VersionedResources[version.Version] = resources.APIResources + } + + // Update information for group resources about the API group by adding new versions. + // Ignore the versions that are already registered. + for _, version := range versions { + found := false + for _, v := range groupResources.Group.Versions { + if v.Version == version { + found = true + break + } + } + + if !found { + groupResources.Group.Versions = append(groupResources.Group.Versions, metav1.GroupVersionForDiscovery{ + GroupVersion: metav1.GroupVersion{Group: groupName, Version: version}.String(), + Version: version, + }) + } + } + + // Update data in the cache. + m.knownGroups[groupName] = groupResources + + // Finally, update the group with received information and regenerate the mapper. + updatedGroupResources := make([]*restmapper.APIGroupResources, 0, len(m.knownGroups)) + for _, agr := range m.knownGroups { + updatedGroupResources = append(updatedGroupResources, agr) + } + + m.mapper = restmapper.NewDiscoveryRESTMapper(updatedGroupResources) + + return nil +} + +// findAPIGroupByNameLocked returns API group by its name. +func (m *lazyRESTMapper) findAPIGroupByNameLocked(groupName string) (metav1.APIGroup, error) { + // Looking in the cache first. + for _, apiGroup := range m.apiGroups { + if groupName == apiGroup.Name { + return apiGroup, nil + } + } + + // Update the cache if nothing was found. + apiGroups, err := m.client.ServerGroups() + if err != nil { + return metav1.APIGroup{}, fmt.Errorf("failed to get server groups: %w", err) + } + if len(apiGroups.Groups) == 0 { + return metav1.APIGroup{}, fmt.Errorf("received an empty API groups list") + } + + m.apiGroups = apiGroups.Groups + + // Looking in the cache again. + for _, apiGroup := range m.apiGroups { + if groupName == apiGroup.Name { + return apiGroup, nil + } + } + + // If there is still nothing, return an error. + return metav1.APIGroup{}, fmt.Errorf("failed to find API group %s", groupName) +} + +// fetchGroupVersionResources fetches the resources for the specified group and its versions. +func (m *lazyRESTMapper) fetchGroupVersionResources(groupName string, versions ...string) (map[schema.GroupVersion]*metav1.APIResourceList, error) { + groupVersionResources := make(map[schema.GroupVersion]*metav1.APIResourceList) + failedGroups := make(map[schema.GroupVersion]error) + + for _, version := range versions { + groupVersion := schema.GroupVersion{Group: groupName, Version: version} + + apiResourceList, err := m.client.ServerResourcesForGroupVersion(groupVersion.String()) + if err != nil { + failedGroups[groupVersion] = err + } + if apiResourceList != nil { + // even in case of error, some fallback might have been returned. + groupVersionResources[groupVersion] = apiResourceList + } + } + + if len(failedGroups) > 0 { + return nil, &discovery.ErrGroupDiscoveryFailed{Groups: failedGroups} + } + + return groupVersionResources, nil +}