UPSTREAM: 47347: actually check for a live discovery endpoint before …

…aggregating

staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go

@@ -233,6 +233,8 @@ func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.Deleg
 		c.CoreKubeInformers.Core().V1().Services(),
 		c.CoreKubeInformers.Core().V1().Endpoints(),
 		apiregistrationClient.Apiregistration(),
+		c.ProxyTransport != nil,
+		s.serviceResolver,
 	)
 
 	s.GenericAPIServer.AddPostStartHook("start-kube-aggregator-informers", func(context genericapiserver.PostStartHookContext) error {

staging/src/k8s.io/kube-aggregator/pkg/controllers/status/available_controller.go

@@ -17,7 +17,10 @@ limitations under the License.
 package apiserver
 
 import (
+	"crypto/tls"
 	"fmt"
+	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/golang/glog"
@@ -45,6 +48,10 @@ import (
 
 var cloner = conversion.NewCloner()
 
+type ServiceResolver interface {
+	ResolveEndpoint(namespace, name string) (*url.URL, error)
+}
+
 type AvailableConditionController struct {
 	apiServiceClient apiregistrationclient.APIServicesGetter
 
@@ -58,6 +65,9 @@ type AvailableConditionController struct {
 	endpointsLister v1listers.EndpointsLister
 	endpointsSynced cache.InformerSynced
 
+	enableAggregatorRouting bool
+	serviceResolver         ServiceResolver
+
 	// To allow injection for testing.
 	syncFn func(key string) error
 
@@ -69,16 +79,20 @@ func NewAvailableConditionController(
 	serviceInformer v1informers.ServiceInformer,
 	endpointsInformer v1informers.EndpointsInformer,
 	apiServiceClient apiregistrationclient.APIServicesGetter,
+	enableAggregatorRouting bool,
+	serviceResolver ServiceResolver,
 ) *AvailableConditionController {
 	c := &AvailableConditionController{
-		apiServiceClient: apiServiceClient,
-		apiServiceLister: apiServiceInformer.Lister(),
-		apiServiceSynced: apiServiceInformer.Informer().HasSynced,
-		serviceLister:    serviceInformer.Lister(),
-		servicesSynced:   serviceInformer.Informer().HasSynced,
-		endpointsLister:  endpointsInformer.Lister(),
-		endpointsSynced:  endpointsInformer.Informer().HasSynced,
-		queue:            workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "AvailableConditionController"),
+		apiServiceClient:        apiServiceClient,
+		apiServiceLister:        apiServiceInformer.Lister(),
+		apiServiceSynced:        apiServiceInformer.Informer().HasSynced,
+		serviceLister:           serviceInformer.Lister(),
+		servicesSynced:          serviceInformer.Informer().HasSynced,
+		endpointsLister:         endpointsInformer.Lister(),
+		endpointsSynced:         endpointsInformer.Informer().HasSynced,
+		enableAggregatorRouting: enableAggregatorRouting,
+		serviceResolver:         serviceResolver,
+		queue:                   workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "AvailableConditionController"),
 	}
 
 	apiServiceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
@@ -184,8 +198,52 @@ func (c *AvailableConditionController) sync(key string) error {
 			return err
 		}
 	}
+	// actually try to hit the discovery endpoint when it isn't local and when we're routing as a service.
+	// TODO figure out if the kube-proxy code handling has similar problems
+	if apiService.Spec.Service != nil && !c.enableAggregatorRouting && c.serviceResolver != nil {
+		discoveryURL, err := c.serviceResolver.ResolveEndpoint(apiService.Spec.Service.Namespace, apiService.Spec.Service.Name)
+		if err != nil {
+			return err
+		}
+		// construct an http client that will ignore TLS verification (if someone owns the network and messes with your status
+		// that's not so bad) and sets a very short timeout.
+		httpClient := &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			},
+			// the request should happen quickly.
+			Timeout: 5 * time.Second,
+		}
+
+		errCh := make(chan error)
+		go func() {
+			resp, err := httpClient.Get(discoveryURL.String())
+			if resp != nil {
+				resp.Body.Close()
+			}
+			errCh <- err
+		}()
 
-	// TODO actually try to hit the discovery endpoint
+		select {
+		case err = <-errCh:
+		case <-time.After(6 * time.Second):
+			err = fmt.Errorf("timed out waiting for %v", discoveryURL)
+		}
+
+		if err != nil {
+			availableCondition.Status = apiregistration.ConditionFalse
+			availableCondition.Reason = "FailedDiscoveryCheck"
+			availableCondition.Message = fmt.Sprintf("no response from %v: %v", discoveryURL, err)
+			apiregistration.SetAPIServiceCondition(apiService, availableCondition)
+			_, updateErr := c.apiServiceClient.APIServices().UpdateStatus(apiService)
+			if updateErr != nil {
+				return updateErr
+			}
+			// force a requeue to make it very obvious that this will be retried at some point in the future
+			// along with other requeues done via service change, endpoint change, and resync
+			return err
+		}
+	}
 
 	availableCondition.Reason = "Passed"
 	availableCondition.Message = "all checks passed"