From 64bc44db53b253a405769404c36dc4243557f382 Mon Sep 17 00:00:00 2001 From: Lukasz Szaszkiewicz Date: Thu, 4 Mar 2021 09:58:56 +0100 Subject: [PATCH] UPSTREAM: : allows for switching KCM to talk to Kube API over localhost to force KCM to use localhost set the following flag in kubecontrollermanager (oc edit kubecontrollermanager cluster) unsupportedConfigOverrides: extendedArguments: unsupported-kube-api-over-localhost: - "true" --- .../app/config/patch.go | 9 ++++ .../app/controllermanager.go | 20 +++++++- .../app/options/options.go | 8 +++ cmd/kube-controller-manager/app/patch.go | 51 +++++++++++++++++++ 4 files changed, 87 insertions(+), 1 deletion(-) diff --git a/cmd/kube-controller-manager/app/config/patch.go b/cmd/kube-controller-manager/app/config/patch.go index a7112d003d41..38b64663b184 100644 --- a/cmd/kube-controller-manager/app/config/patch.go +++ b/cmd/kube-controller-manager/app/config/patch.go @@ -1,9 +1,18 @@ package config +import ( + "k8s.io/client-go/transport" + + "github.com/openshift/library-go/pkg/monitor/health" +) + // OpenShiftContext is additional context that we need to launch the kube-controller-manager for openshift. // Basically, this holds our additional config information. type OpenShiftContext struct { OpenShiftConfig string OpenShiftDefaultProjectNodeSelector string KubeDefaultProjectNodeSelector string + UnsupportedKubeAPIOverPreferredHost bool + PreferredHostRoundTripperWrapperFn transport.WrapperFunc + PreferredHostHealthMonitor *health.Prober } diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index cb5992288f87..6de6c5fc04c3 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -70,6 +70,8 @@ import ( kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config" serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount" "k8s.io/kubernetes/pkg/serviceaccount" + + libgorestclient "github.com/openshift/library-go/pkg/config/client" ) const ( @@ -134,6 +136,11 @@ controller, and serviceaccounts controller.`, os.Exit(1) } + if err := SetUpPreferredHostForOpenShift(s); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List()) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) @@ -197,6 +204,17 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { klog.Errorf("unable to register configz: %v", err) } + // start the localhost health monitor early so that it can be used by the LE client + if c.OpenShiftContext.PreferredHostHealthMonitor != nil { + hmCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + <-stopCh + cancel() + }() + go c.OpenShiftContext.PreferredHostHealthMonitor.Run(hmCtx) + } + // Setup any healthz checks we will want to use. var checks []healthz.HealthChecker var electionChecker *leaderelection.HealthzAdaptor @@ -695,7 +713,7 @@ func createClientBuilders(c *config.CompletedConfig) (clientBuilder clientbuilde } clientBuilder = clientbuilder.NewDynamicClientBuilder( - restclient.AnonymousClientConfig(c.Kubeconfig), + libgorestclient.AnonymousClientConfigWithWrapTransport(c.Kubeconfig), c.Client.CoreV1(), metav1.NamespaceSystem) } else { diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 5a80fa1f88d3..ae451f7692d9 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -48,6 +48,8 @@ import ( // add the kubernetes feature gates _ "k8s.io/kubernetes/pkg/features" + + libgorestclient "github.com/openshift/library-go/pkg/config/client" ) const ( @@ -278,6 +280,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy fs.StringVar(&dummy, "insecure-experimental-approve-all-kubelet-csrs-for-group", "", "This flag does nothing.") fs.StringVar(&s.OpenShiftContext.OpenShiftConfig, "openshift-config", s.OpenShiftContext.OpenShiftConfig, "indicates that this process should be compatible with openshift start master") fs.MarkHidden("openshift-config") + fs.BoolVar(&s.OpenShiftContext.UnsupportedKubeAPIOverPreferredHost, "unsupported-kube-api-over-localhost", false, "when set makes KCM prefer talking to localhost kube-apiserver (when available) instead of LB") utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic")) return fss @@ -441,6 +444,11 @@ func (s KubeControllerManagerOptions) Config(allControllers []string, disabledBy kubeconfig.QPS = s.Generic.ClientConnection.QPS kubeconfig.Burst = int(s.Generic.ClientConnection.Burst) + if s.OpenShiftContext.PreferredHostRoundTripperWrapperFn != nil { + libgorestclient.DefaultServerName(kubeconfig) + kubeconfig.Wrap(s.OpenShiftContext.PreferredHostRoundTripperWrapperFn) + } + client, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, KubeControllerManagerUserAgent)) if err != nil { return nil, err diff --git a/cmd/kube-controller-manager/app/patch.go b/cmd/kube-controller-manager/app/patch.go index 5d85c0229931..527ae0fcc7c9 100644 --- a/cmd/kube-controller-manager/app/patch.go +++ b/cmd/kube-controller-manager/app/patch.go @@ -1,18 +1,62 @@ package app import ( + "fmt" "io/ioutil" "path" + "time" "k8s.io/apimachinery/pkg/util/json" kyaml "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/informers" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/component-base/metrics/legacyregistry" "k8s.io/kubernetes/cmd/kube-controller-manager/app/config" "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" + + libgorestclient "github.com/openshift/library-go/pkg/config/client" + "github.com/openshift/library-go/pkg/monitor/health" ) var InformerFactoryOverride informers.SharedInformerFactory +func SetUpPreferredHostForOpenShift(controllerManagerOptions *options.KubeControllerManagerOptions) error { + if !controllerManagerOptions.OpenShiftContext.UnsupportedKubeAPIOverPreferredHost { + return nil + } + + config, err := clientcmd.BuildConfigFromFlags(controllerManagerOptions.Master, controllerManagerOptions.Kubeconfig) + if err != nil { + return err + } + libgorestclient.DefaultServerName(config) + + targetProvider := health.StaticTargetProvider{"localhost:6443"} + controllerManagerOptions.OpenShiftContext.PreferredHostHealthMonitor, err = health.New(targetProvider, createRestConfigForHealthMonitor(config)) + if err != nil { + return err + } + controllerManagerOptions.OpenShiftContext.PreferredHostHealthMonitor. + WithHealthyProbesThreshold(3). + WithUnHealthyProbesThreshold(5). + WithProbeInterval(5 * time.Second). + WithProbeResponseTimeout(2 * time.Second). + WithMetrics(health.Register(legacyregistry.MustRegister)) + + controllerManagerOptions.OpenShiftContext.PreferredHostRoundTripperWrapperFn = libgorestclient.NewPreferredHostRoundTripper(func() string { + healthyTargets, _ := controllerManagerOptions.OpenShiftContext.PreferredHostHealthMonitor.Targets() + if len(healthyTargets) == 1 { + return healthyTargets[0] + } + return "" + }) + + controllerManagerOptions.Authentication.WithCustomRoundTripper(controllerManagerOptions.OpenShiftContext.PreferredHostRoundTripperWrapperFn) + controllerManagerOptions.Authorization.WithCustomRoundTripper(controllerManagerOptions.OpenShiftContext.PreferredHostRoundTripperWrapperFn) + return nil +} + func ShimForOpenShift(controllerManagerOptions *options.KubeControllerManagerOptions, controllerManager *config.Config) error { if len(controllerManager.OpenShiftContext.OpenShiftConfig) == 0 { return nil @@ -82,3 +126,10 @@ func applyOpenShiftConfigDefaultProjectSelector(controllerManagerOptions *option return nil } + +func createRestConfigForHealthMonitor(restConfig *rest.Config) *rest.Config { + restConfigCopy := *restConfig + rest.AddUserAgent(&restConfigCopy, fmt.Sprintf("%s-health-monitor", options.KubeControllerManagerUserAgent)) + + return &restConfigCopy +}