Bug 1985997: wires the startup monitor

cmd/cluster-kube-apiserver-operator/main.go

@@ -11,6 +11,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
+	"k8s.io/client-go/rest"
 	utilflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/logs"
 
@@ -21,10 +22,14 @@ import (
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/cmd/render"
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/cmd/resourcegraph"
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator"
+	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/startupmonitorreadiness"
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/version"
 	"github.com/openshift/library-go/pkg/operator/staticpod/certsyncpod"
 	"github.com/openshift/library-go/pkg/operator/staticpod/installerpod"
 	"github.com/openshift/library-go/pkg/operator/staticpod/prune"
+	"github.com/openshift/library-go/pkg/operator/staticpod/startupmonitor"
+
+	operatorclientv1 "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1"
 )
 
 func main() {
@@ -68,6 +73,13 @@ func NewOperatorCommand(ctx context.Context) *cobra.Command {
 	cmd.AddCommand(certregenerationcontroller.NewCertRegenerationControllerCommand(ctx))
 	cmd.AddCommand(insecurereadyz.NewInsecureReadyzCommand())
 	cmd.AddCommand(checkendpoints.NewCheckEndpointsCommand())
+	cmd.AddCommand(startupmonitor.NewCommand(startupmonitorreadiness.New(), func(config *rest.Config) (operatorclientv1.KubeAPIServerInterface, error) {
+		client, err := operatorclientv1.NewForConfig(config)
+		if err != nil {
+			return nil, err
+		}
+		return client.KubeAPIServers(), nil
+	}))
 
 	return cmd
 }

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/openshift/api v0.0.0-20210706092853-b63d499a70ce
 	github.com/openshift/build-machinery-go v0.0.0-20210423112049-9415d7ebd33e
 	github.com/openshift/client-go v0.0.0-20210521082421-73d9475a9142
-	github.com/openshift/library-go v0.0.0-20210723094744-7b9501f356ce
+	github.com/openshift/library-go v0.0.0-20210723141450-212eeb60ef3e
 	github.com/pkg/profile v1.5.0 // indirect
 	github.com/prometheus-operator/prometheus-operator/pkg/client v0.45.0
 	github.com/prometheus/client_golang v1.7.1

go.sum

@@ -421,8 +421,8 @@ github.com/openshift/client-go v0.0.0-20210521082421-73d9475a9142 h1:ZHRIMCFIJN1
 github.com/openshift/client-go v0.0.0-20210521082421-73d9475a9142/go.mod h1:fjS8r9mqDVsPb5td3NehsNOAWa4uiFkYEfVZioQ2gH0=
 github.com/openshift/kubernetes-apiserver v0.0.0-20210419140141-620426e63a99 h1:KrCYRAJcgZYzMCB1PjJHJMYPu/d+dEkelq5eYyi0fDw=
 github.com/openshift/kubernetes-apiserver v0.0.0-20210419140141-620426e63a99/go.mod h1:w2YSn4/WIwYuxG5zJmcqtRdtqgW/J2JRgFAqps3bBpg=
-github.com/openshift/library-go v0.0.0-20210723094744-7b9501f356ce h1:39jq3DTPT3Uh8s8c9c23l25aKg1NL7xP2UAea2x5apQ=
-github.com/openshift/library-go v0.0.0-20210723094744-7b9501f356ce/go.mod h1:rln3LbFNOpENSvhmsfH7g/hqc58IF78+o96yAAp5mq0=
+github.com/openshift/library-go v0.0.0-20210723141450-212eeb60ef3e h1:Af3yumNDfCv7F0Esw+h9gk8efiPKTwKc+xtvmGRWa34=
+github.com/openshift/library-go v0.0.0-20210723141450-212eeb60ef3e/go.mod h1:rln3LbFNOpENSvhmsfH7g/hqc58IF78+o96yAAp5mq0=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=

pkg/operator/starter.go

@@ -165,6 +165,7 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
 		operatorClient,
 		kubeInformersForNamespaces.InformersFor(operatorclient.TargetNamespace),
 		kubeInformersForNamespaces,
+		configInformers.Config().V1().Infrastructures(),
 		kubeClient,
 		controllerContext.EventRecorder,
 	)

pkg/operator/targetconfigcontroller/targetconfigcontroller.go

@@ -13,8 +13,11 @@ import (
 
 	"github.com/ghodss/yaml"
 
+	configv1 "github.com/openshift/api/config/v1"
 	kubecontrolplanev1 "github.com/openshift/api/kubecontrolplane/v1"
 	operatorv1 "github.com/openshift/api/operator/v1"
+	configinformersv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1"
+	configlistersv1 "github.com/openshift/client-go/config/listers/config/v1"
 	"github.com/openshift/cluster-kube-apiserver-operator/bindata"
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/operatorclient"
 	"github.com/openshift/cluster-kube-apiserver-operator/pkg/version"
@@ -24,6 +27,7 @@ import (
 	"github.com/openshift/library-go/pkg/operator/resource/resourcemerge"
 	"github.com/openshift/library-go/pkg/operator/resource/resourceread"
 	"github.com/openshift/library-go/pkg/operator/resourcesynccontroller"
+	"github.com/openshift/library-go/pkg/operator/staticpod/startupmonitor"
 	"github.com/openshift/library-go/pkg/operator/v1helpers"
 
 	corev1 "k8s.io/api/core/v1"
@@ -42,15 +46,17 @@ type TargetConfigController struct {
 
 	operatorClient v1helpers.StaticPodOperatorClient
 
-	kubeClient      kubernetes.Interface
-	configMapLister corev1listers.ConfigMapLister
+	kubeClient           kubernetes.Interface
+	configMapLister      corev1listers.ConfigMapLister
+	infrastructureLister configlistersv1.InfrastructureLister
 }
 
 func NewTargetConfigController(
 	targetImagePullSpec, operatorImagePullSpec string,
 	operatorClient v1helpers.StaticPodOperatorClient,
 	kubeInformersForOpenshiftKubeAPIServerNamespace informers.SharedInformerFactory,
 	kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces,
+	infrastructureInformer configinformersv1.InfrastructureInformer,
 	kubeClient kubernetes.Interface,
 	eventRecorder events.Recorder,
 ) factory.Controller {
@@ -60,10 +66,12 @@ func NewTargetConfigController(
 		operatorClient:        operatorClient,
 		kubeClient:            kubeClient,
 		configMapLister:       kubeInformersForNamespaces.ConfigMapLister(),
+		infrastructureLister:  infrastructureInformer.Lister(),
 	}
 
 	return factory.New().WithInformers(
 		operatorClient.Informer(),
+		infrastructureInformer.Informer(),
 		kubeInformersForOpenshiftKubeAPIServerNamespace.Core().V1().ConfigMaps().Informer(),
 		kubeInformersForOpenshiftKubeAPIServerNamespace.Core().V1().Secrets().Informer(),
 		kubeInformersForOpenshiftKubeAPIServerNamespace.Core().V1().ServiceAccounts().Informer(),
@@ -157,7 +165,7 @@ func createTargetConfig(ctx context.Context, c TargetConfigController, recorder
 	if err != nil {
 		errors = append(errors, fmt.Errorf("%q: %v", "configmap/config", err))
 	}
-	_, _, err = managePod(ctx, c.kubeClient.CoreV1(), recorder, operatorSpec, c.targetImagePullSpec, c.operatorImagePullSpec)
+	_, _, err = managePods(ctx, c.kubeClient.CoreV1(), c.infrastructureLister, recorder, operatorSpec, c.targetImagePullSpec, c.operatorImagePullSpec)
 	if err != nil {
 		errors = append(errors, fmt.Errorf("%q: %v", "configmap/kube-apiserver-pod", err))
 	}
@@ -226,7 +234,7 @@ func manageKubeAPIServerConfig(ctx context.Context, client coreclientv1.ConfigMa
 	return resourceapply.ApplyConfigMap(ctx, client, recorder, requiredConfigMap)
 }
 
-func managePod(ctx context.Context, client coreclientv1.ConfigMapsGetter, recorder events.Recorder, operatorSpec *operatorv1.StaticPodOperatorSpec, imagePullSpec, operatorImagePullSpec string) (*corev1.ConfigMap, bool, error) {
+func managePods(ctx context.Context, client coreclientv1.ConfigMapsGetter, infrastructureLister configlistersv1.InfrastructureLister, recorder events.Recorder, operatorSpec *operatorv1.StaticPodOperatorSpec, imagePullSpec, operatorImagePullSpec string) (*corev1.ConfigMap, bool, error) {
 	appliedPodTemplate, err := manageTemplate(string(bindata.MustAsset("assets/kube-apiserver/pod.yaml")), imagePullSpec, operatorImagePullSpec, operatorSpec)
 	if err != nil {
 		return nil, false, err
@@ -251,9 +259,49 @@ func managePod(ctx context.Context, client coreclientv1.ConfigMapsGetter, record
 	configMap.Data["pod.yaml"] = resourceread.WritePodV1OrDie(required)
 	configMap.Data["forceRedeploymentReason"] = operatorSpec.ForceRedeploymentReason
 	configMap.Data["version"] = version.Get().String()
+
+	startupMonitorPodKey, optionalStartupMonitor, err := generateOptionalStartupMonitorPod(infrastructureLister, operatorSpec, operatorImagePullSpec)
+	if err != nil {
+		return nil, false, fmt.Errorf("failed to apply an optional pod due to %v", err)
+	}
+	if optionalStartupMonitor != nil {
+		configMap.Data[startupMonitorPodKey] = resourceread.WritePodV1OrDie(optionalStartupMonitor)
+	}
 	return resourceapply.ApplyConfigMap(ctx, client, recorder, configMap)
 }
 
+func generateOptionalStartupMonitorPod(infrastructureLister configlistersv1.InfrastructureLister, operatorSpec *operatorv1.StaticPodOperatorSpec, operatorImagePullSpec string) (string, *corev1.Pod, error) {
+	infra, err := infrastructureLister.Get("cluster")
+	if err != nil && !apierrors.IsNotFound(err) {
+		// we got an error so without the infrastructure object we are not able to determine the type of platform we are running on
+		return "", nil, err
+	}
+	if infra.Status.ControlPlaneTopology != configv1.SingleReplicaTopologyMode {
+		return "", nil, nil
+	}
+
+	// TODO: remove before releasing 4.9
+	startupMonitorExplicitlyEnabled := false
+	if len(operatorSpec.OperatorSpec.UnsupportedConfigOverrides.Raw) > 0 {
+		observedUnsupportedConfig := map[string]interface{}{}
+		if err := json.NewDecoder(bytes.NewBuffer(operatorSpec.UnsupportedConfigOverrides.Raw)).Decode(&observedUnsupportedConfig); err != nil {
+			return "", nil, err
+		}
+		startupMonitorExplicitlyEnabled, _, _ = unstructured.NestedBool(observedUnsupportedConfig, "startupMonitor")
+	}
+	if !startupMonitorExplicitlyEnabled {
+		return "", nil, nil
+	}
+	// End of TODO
+
+	appliedStartupMonitorPodTemplate, err := startupmonitor.GeneratePodTemplate(operatorSpec, []string{"cluster-kube-apiserver-operator", "startup-monitor"}, operatorclient.TargetNamespace, "kube-apiserver", operatorImagePullSpec)
+	if err != nil {
+		return "", nil, err
+	}
+	required := resourceread.ReadPodV1OrDie([]byte(appliedStartupMonitorPodTemplate))
+	return "kube-apiserver-startup-monitor-pod.yaml", required, nil
+}
+
 func ManageClientCABundle(ctx context.Context, lister corev1listers.ConfigMapLister, client coreclientv1.ConfigMapsGetter, recorder events.Recorder) (*corev1.ConfigMap, bool, error) {
 	requiredConfigMap, err := resourcesynccontroller.CombineCABundleConfigMaps(
 		resourcesynccontroller.ResourceLocation{Namespace: operatorclient.TargetNamespace, Name: "client-ca"},