UPSTREAM: <carry>: allows for switching KCM to talk to Kube API over …

…localhost

to force KCM to use localhost set the following flag in kubecontrollermanager (oc edit kubecontrollermanager cluster)

unsupportedConfigOverrides:
  extendedArguments:
    unsupported-kube-api-over-localhost:
      - "true"

cmd/kube-controller-manager/app/config/patch.go

@@ -1,9 +1,18 @@
 package config
 
+import (
+	"k8s.io/client-go/transport"
+
+	"github.com/openshift/library-go/pkg/monitor/health"
+)
+
 // OpenShiftContext is additional context that we need to launch the kube-controller-manager for openshift.
 // Basically, this holds our additional config information.
 type OpenShiftContext struct {
 	OpenShiftConfig                     string
 	OpenShiftDefaultProjectNodeSelector string
 	KubeDefaultProjectNodeSelector      string
+	UnsupportedKubeAPIOverPreferredHost bool
+	PreferredHostRoundTripperWrapperFn  transport.WrapperFunc
+	PreferredHostHealthMonitor          *health.Prober
 }

cmd/kube-controller-manager/app/controllermanager.go

@@ -70,6 +70,8 @@ import (
 	kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config"
 	serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
 	"k8s.io/kubernetes/pkg/serviceaccount"
+
+	libgorestclient "github.com/openshift/library-go/pkg/config/client"
 )
 
 const (
@@ -134,6 +136,11 @@ controller, and serviceaccounts controller.`,
 				os.Exit(1)
 			}
 
+			if err := SetUpPreferredHostForOpenShift(s); err != nil {
+				fmt.Fprintf(os.Stderr, "%v\n", err)
+				os.Exit(1)
+			}
+
 			c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List())
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "%v\n", err)
@@ -197,6 +204,17 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {
 		klog.Errorf("unable to register configz: %v", err)
 	}
 
+	// start the localhost health monitor early so that it can be used by the LE client
+	if c.OpenShiftContext.PreferredHostHealthMonitor != nil {
+		hmCtx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		go func() {
+			<-stopCh
+			cancel()
+		}()
+		go c.OpenShiftContext.PreferredHostHealthMonitor.Run(hmCtx)
+	}
+
 	// Setup any healthz checks we will want to use.
 	var checks []healthz.HealthChecker
 	var electionChecker *leaderelection.HealthzAdaptor
@@ -695,7 +713,7 @@ func createClientBuilders(c *config.CompletedConfig) (clientBuilder clientbuilde
 		}
 
 		clientBuilder = clientbuilder.NewDynamicClientBuilder(
-			restclient.AnonymousClientConfig(c.Kubeconfig),
+			libgorestclient.AnonymousClientConfigWithWrapTransport(c.Kubeconfig),
 			c.Client.CoreV1(),
 			metav1.NamespaceSystem)
 	} else {

cmd/kube-controller-manager/app/options/options.go

@@ -48,6 +48,8 @@ import (
 
 	// add the kubernetes feature gates
 	_ "k8s.io/kubernetes/pkg/features"
+
+	libgorestclient "github.com/openshift/library-go/pkg/config/client"
 )
 
 const (
@@ -278,6 +280,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy
 	fs.StringVar(&dummy, "insecure-experimental-approve-all-kubelet-csrs-for-group", "", "This flag does nothing.")
 	fs.StringVar(&s.OpenShiftContext.OpenShiftConfig, "openshift-config", s.OpenShiftContext.OpenShiftConfig, "indicates that this process should be compatible with openshift start master")
 	fs.MarkHidden("openshift-config")
+	fs.BoolVar(&s.OpenShiftContext.UnsupportedKubeAPIOverPreferredHost, "unsupported-kube-api-over-localhost", false, "when set makes KCM prefer talking to localhost kube-apiserver (when available) instead of LB")
 	utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))
 
 	return fss
@@ -441,6 +444,11 @@ func (s KubeControllerManagerOptions) Config(allControllers []string, disabledBy
 	kubeconfig.QPS = s.Generic.ClientConnection.QPS
 	kubeconfig.Burst = int(s.Generic.ClientConnection.Burst)
 
+	if s.OpenShiftContext.PreferredHostRoundTripperWrapperFn != nil {
+		libgorestclient.DefaultServerName(kubeconfig)
+		kubeconfig.Wrap(s.OpenShiftContext.PreferredHostRoundTripperWrapperFn)
+	}
+
 	client, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, KubeControllerManagerUserAgent))
 	if err != nil {
 		return nil, err

cmd/kube-controller-manager/app/patch.go

@@ -1,18 +1,62 @@
 package app
 
 import (
+	"fmt"
 	"io/ioutil"
 	"path"
+	"time"
 
 	"k8s.io/apimachinery/pkg/util/json"
 	kyaml "k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/client-go/informers"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/component-base/metrics/legacyregistry"
 	"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
 	"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
+
+	libgorestclient "github.com/openshift/library-go/pkg/config/client"
+	"github.com/openshift/library-go/pkg/monitor/health"
 )
 
 var InformerFactoryOverride informers.SharedInformerFactory
 
+func SetUpPreferredHostForOpenShift(controllerManagerOptions *options.KubeControllerManagerOptions) error {
+	if !controllerManagerOptions.OpenShiftContext.UnsupportedKubeAPIOverPreferredHost {
+		return nil
+	}
+
+	config, err := clientcmd.BuildConfigFromFlags(controllerManagerOptions.Master, controllerManagerOptions.Kubeconfig)
+	if err != nil {
+		return err
+	}
+	libgorestclient.DefaultServerName(config)
+
+	targetProvider := health.StaticTargetProvider{"localhost:6443"}
+	controllerManagerOptions.OpenShiftContext.PreferredHostHealthMonitor, err = health.New(targetProvider, createRestConfigForHealthMonitor(config))
+	if err != nil {
+		return err
+	}
+	controllerManagerOptions.OpenShiftContext.PreferredHostHealthMonitor.
+		WithHealthyProbesThreshold(3).
+		WithUnHealthyProbesThreshold(5).
+		WithProbeInterval(5 * time.Second).
+		WithProbeResponseTimeout(2 * time.Second).
+		WithMetrics(health.Register(legacyregistry.MustRegister))
+
+	controllerManagerOptions.OpenShiftContext.PreferredHostRoundTripperWrapperFn = libgorestclient.NewPreferredHostRoundTripper(func() string {
+		healthyTargets, _ := controllerManagerOptions.OpenShiftContext.PreferredHostHealthMonitor.Targets()
+		if len(healthyTargets) == 1 {
+			return healthyTargets[0]
+		}
+		return ""
+	})
+
+	controllerManagerOptions.Authentication.WithCustomRoundTripper(controllerManagerOptions.OpenShiftContext.PreferredHostRoundTripperWrapperFn)
+	controllerManagerOptions.Authorization.WithCustomRoundTripper(controllerManagerOptions.OpenShiftContext.PreferredHostRoundTripperWrapperFn)
+	return nil
+}
+
 func ShimForOpenShift(controllerManagerOptions *options.KubeControllerManagerOptions, controllerManager *config.Config) error {
 	if len(controllerManager.OpenShiftContext.OpenShiftConfig) == 0 {
 		return nil
@@ -82,3 +126,10 @@ func applyOpenShiftConfigDefaultProjectSelector(controllerManagerOptions *option
 
 	return nil
 }
+
+func createRestConfigForHealthMonitor(restConfig *rest.Config) *rest.Config {
+	restConfigCopy := *restConfig
+	rest.AddUserAgent(&restConfigCopy, fmt.Sprintf("%s-health-monitor", options.KubeControllerManagerUserAgent))
+
+	return &restConfigCopy
+}