pkg/manifests: Deploy metrics-server through operator

Signed-off-by: Jayapriya Pai <janantha@redhat.com>

Documentation/api.md

@@ -25,6 +25,7 @@ Configuring Cluster Monitoring is optional. If the config does not exist or is e
 * [DedicatedServiceMonitors](#dedicatedservicemonitors)
 * [K8sPrometheusAdapter](#k8sprometheusadapter)
 * [KubeStateMetricsConfig](#kubestatemetricsconfig)
+* [MetricsServerConfig](#metricsserverconfig)
 * [MonitoringPluginConfig](#monitoringpluginconfig)
 * [NodeExporterCollectorBuddyInfoConfig](#nodeexportercollectorbuddyinfoconfig)
 * [NodeExporterCollectorConfig](#nodeexportercollectorconfig)
@@ -128,6 +129,7 @@ The `ClusterMonitoringConfiguration` resource defines settings that customize th
 | alertmanagerMain | *[AlertmanagerMainConfig](#alertmanagermainconfig) | `AlertmanagerMainConfig` defines settings for the Alertmanager component in the `openshift-monitoring` namespace. |
 | enableUserWorkload | *bool | `UserWorkloadEnabled` is a Boolean flag that enables monitoring for user-defined projects. |
 | k8sPrometheusAdapter | *[K8sPrometheusAdapter](#k8sprometheusadapter) | `K8sPrometheusAdapter` defines settings for the Prometheus Adapter component. |
+| metricsServer | *[MetricsServerConfig](#metricsserverconfig) | `MetricsServer` defines settings for the MetricsServer component. |
 | kubeStateMetrics | *[KubeStateMetricsConfig](#kubestatemetricsconfig) | `KubeStateMetricsConfig` defines settings for the `kube-state-metrics` agent. |
 | prometheusK8s | *[PrometheusK8sConfig](#prometheusk8sconfig) | `PrometheusK8sConfig` defines settings for the Prometheus component. |
 | prometheusOperator | *[PrometheusOperatorConfig](#prometheusoperatorconfig) | `PrometheusOperatorConfig` defines settings for the Prometheus Operator component. |
@@ -190,6 +192,24 @@ The `KubeStateMetricsConfig` resource defines settings for the `kube-state-metri
 
 [Back to TOC](#table-of-contents)
 
+## MetricsServerConfig
+
+#### Description
+
+The `MetricsServerConfig` resource defines settings for the MetricsServer component.
+
+
+<em>appears in: [ClusterMonitoringConfiguration](#clustermonitoringconfiguration)</em>
+
+| Property | Type | Description |
+| -------- | ---- | ----------- |
+| enabled | *bool | `Enabled` is a Boolean flag that enables or disables metrics-server. |
+| nodeSelector | map[string]string | Defines the nodes on which the pods are scheduled. |
+| tolerations | [][v1.Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#toleration-v1-core) | Defines tolerations for the pods. |
+| resources | *[v1.ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#resourcerequirements-v1-core) | Defines resource requests and limits for the Metrics Server container. |
+
+[Back to TOC](#table-of-contents)
+
 ## MonitoringPluginConfig
 
 #### Description

Documentation/openshiftdocs/index.adoc

@@ -45,6 +45,7 @@ The configuration file itself is always defined under the `config.yaml` key in t
 * link:modules/dedicatedservicemonitors.adoc[DedicatedServiceMonitors]
 * link:modules/k8sprometheusadapter.adoc[K8sPrometheusAdapter]
 * link:modules/kubestatemetricsconfig.adoc[KubeStateMetricsConfig]
+* link:modules/metricsserverconfig.adoc[MetricsServerConfig]
 * link:modules/monitoringpluginconfig.adoc[MonitoringPluginConfig]
 * link:modules/nodeexportercollectorbuddyinfoconfig.adoc[NodeExporterCollectorBuddyInfoConfig]
 * link:modules/nodeexportercollectorconfig.adoc[NodeExporterCollectorConfig]

Documentation/openshiftdocs/modules/clustermonitoringconfiguration.adoc

@@ -21,6 +21,8 @@ The `ClusterMonitoringConfiguration` resource defines settings that customize th
 
 |k8sPrometheusAdapter|*link:k8sprometheusadapter.adoc[K8sPrometheusAdapter]|`K8sPrometheusAdapter` defines settings for the Prometheus Adapter component.
 
+|metricsServer|*link:metricsserverconfig.adoc[MetricsServerConfig]|`MetricsServer` defines settings for the MetricsServer component.
+
 |kubeStateMetrics|*link:kubestatemetricsconfig.adoc[KubeStateMetricsConfig]|`KubeStateMetricsConfig` defines settings for the `kube-state-metrics` agent.
 
 |prometheusK8s|*link:prometheusk8sconfig.adoc[PrometheusK8sConfig]|`PrometheusK8sConfig` defines settings for the Prometheus component.

Documentation/openshiftdocs/modules/metricsserverconfig.adoc

@@ -0,0 +1,31 @@
+// DO NOT EDIT THE CONTENT IN THIS FILE. It is automatically generated from the 
+	// source code for the Cluster Monitoring Operator. Any changes made to this 
+	// file will be overwritten when the content is re-generated. If you wish to 
+	// make edits, read the docgen utility instructions in the source code for the 
+	// CMO.
+	:_content-type: ASSEMBLY
+
+== MetricsServerConfig
+
+=== Description
+
+The `MetricsServerConfig` resource defines settings for the MetricsServer component.
+
+
+
+Appears in: link:clustermonitoringconfiguration.adoc[ClusterMonitoringConfiguration]
+
+[options="header"]
+|===
+| Property | Type | Description 
+|enabled|*bool|`Enabled` is a Boolean flag that enables or disables metrics-server.
+
+|nodeSelector|map[string]string|Defines the nodes on which the pods are scheduled.
+
+|tolerations|[]v1.Toleration|Defines tolerations for the pods.
+
+|resources|*v1.ResourceRequirements|Defines resource requests and limits for the Metrics Server container.
+
+|===
+
+link:../index.adoc[Back to TOC]

hack/build-jsonnet.sh

@@ -37,7 +37,7 @@ for file in "${files[@]}"; do
 	}&
 
 	# wait for at least one of the jobs to finish if there are more than maxProc jobs
-	while [[ $(jobs -r | wc -l ) -ge "$maxProc" ]]; do wait -n; done
+	while [[ $(jobs -r | wc -l ) -ge "$maxProc" ]]; do wait; done
 done
 # wait for all jobs to finish
 wait

manifests/image-references

@@ -70,3 +70,7 @@ spec:
     from:
       kind: DockerImage
       name: quay.io/openshift/origin-monitoring-plugin:latest
+  - name: kube-metrics-server
+    from:
+      kind: DockerImage
+      name: quay.io/openshift/origin-kube-metrics-server:latest

pkg/manifests/config.go

@@ -128,6 +128,7 @@ func (c Config) GetThanosRulerAlertmanagerConfigs() []AdditionalAlertmanagerConf
 
 type Images struct {
 	K8sPrometheusAdapter               string
+	MetricsServer                      string
 	PromLabelProxy                     string
 	PrometheusOperatorAdmissionWebhook string
 	PrometheusOperator                 string
@@ -208,6 +209,10 @@ func NewConfig(content io.Reader, tp bool) (*Config, error) {
 		return nil, errors.Wrap(ErrConfigValidation, "collectionProfiles is a TechPreview feature, to be able to use a profile different from the default (\"full\") please enable TechPreview")
 	}
 
+	if c.ClusterMonitoringConfiguration.MetricsServerConfig != nil && !tp {
+		return nil, errors.Wrap(ErrConfigValidation, "Metrics Server is a TechPreview feature, to be able to deploy it please enable TechPreview")
+	}
+
 	// Validate the configured collection profile iff tech preview is enabled, even if the default profile is set.
 	if tp {
 		for _, profile := range SupportedCollectionProfiles {
@@ -345,6 +350,8 @@ func (c *Config) SetImages(images map[string]string) {
 	c.Images.TelemeterClient = images["telemeter-client"]
 	c.Images.PromLabelProxy = images["prom-label-proxy"]
 	c.Images.K8sPrometheusAdapter = images["k8s-prometheus-adapter"]
+	c.Images.MetricsServer = images["kube-metrics-server"]
+	// c.Images.MetricsServer = "registry.k8s.io/metrics-server/metrics-server:v0.6.4"
 	c.Images.OpenShiftStateMetrics = images["openshift-state-metrics"]
 	c.Images.Thanos = images["thanos"]
 	c.Images.MonitoringPlugin = images["monitoring-plugin"]

pkg/manifests/manifests.go

@@ -197,6 +197,18 @@ var (
 	PrometheusAdapterMinimalServiceMonitor              = "prometheus-adapter/minimal-service-monitor.yaml"
 	PrometheusAdapterServiceAccount                     = "prometheus-adapter/service-account.yaml"
 
+	MetricsServerAPIService                         = "metrics-server/api-service.yaml"
+	MetricsServerServiceAccount                     = "metrics-server/service-account.yaml"
+	MetricsServerClusterRole                        = "metrics-server/cluster-role.yaml"
+	MetricsServerClusterRoleBinding                 = "metrics-server/cluster-role-binding.yaml"
+	MetricsServerClusterRoleAggregatedMetricsReader = "metrics-server/cluster-role-aggregated-metrics-reader.yaml"
+	MetricsServerClusterRoleBindingAuthDelegator    = "metrics-server/cluster-role-binding-auth-delegator.yaml"
+	MetricsServerRoleBindingAuthReader              = "metrics-server/role-binding-auth-reader.yaml"
+	MetricsServerDeployment                         = "metrics-server/deployment.yaml"
+	MetricsServerService                            = "metrics-server/service.yaml"
+	MetricsServerServiceMonitor                     = "metrics-server/service-monitor.yaml"
+	MetricsServerPodDisruptionBudget                = "metrics-server/pod-disruption-budget.yaml"
+
 	AdmissionWebhookRuleValidatingWebhook               = "admission-webhook/prometheus-rule-validating-webhook.yaml"
 	AdmissionWebhookAlertmanagerConfigValidatingWebhook = "admission-webhook/alertmanager-config-validating-webhook.yaml"
 	AdmissionWebhookDeployment                          = "admission-webhook/deployment.yaml"
@@ -2079,6 +2091,75 @@ func (f *Factory) PrometheusAdapterAPIService() (*apiregistrationv1.APIService,
 	return f.NewAPIService(f.assets.MustNewAssetReader(PrometheusAdapterAPIService))
 }
 
+func (f *Factory) MetricsServerServiceAccount() (*v1.ServiceAccount, error) {
+	return f.NewServiceAccount(f.assets.MustNewAssetReader(MetricsServerServiceAccount))
+}
+
+func (f *Factory) MetricsServerClusterRole() (*rbacv1.ClusterRole, error) {
+	return f.NewClusterRole(f.assets.MustNewAssetReader(MetricsServerClusterRole))
+}
+
+func (f *Factory) MetricsServerClusterRoleBinding() (*rbacv1.ClusterRoleBinding, error) {
+	return f.NewClusterRoleBinding(f.assets.MustNewAssetReader(MetricsServerClusterRoleBinding))
+}
+
+func (f *Factory) MetricsServerClusterRoleAggregatedMetricsReader() (*rbacv1.ClusterRole, error) {
+	return f.NewClusterRole(f.assets.MustNewAssetReader(MetricsServerClusterRoleAggregatedMetricsReader))
+}
+
+func (f *Factory) MetricsServerClusterRoleBindingAuthDelegator() (*rbacv1.ClusterRoleBinding, error) {
+	return f.NewClusterRoleBinding(f.assets.MustNewAssetReader(MetricsServerClusterRoleBindingAuthDelegator))
+}
+
+func (f *Factory) MetricsServerRoleBindingAuthReader() (*rbacv1.RoleBinding, error) {
+	return f.NewRoleBinding(f.assets.MustNewAssetReader(MetricsServerRoleBindingAuthReader))
+}
+
+func (f *Factory) MetricsServerDeployment() (*appsv1.Deployment, error) {
+	d, err := f.NewDeployment(f.assets.MustNewAssetReader(MetricsServerDeployment))
+	if err != nil {
+		return nil, err
+	}
+	for i, container := range d.Spec.Template.Spec.Containers {
+		switch container.Name {
+		case "metrics-server":
+			d.Spec.Template.Spec.Containers[i].Image = f.config.Images.MetricsServer
+
+			if f.config.ClusterMonitoringConfiguration.MetricsServerConfig != nil {
+				if f.config.ClusterMonitoringConfiguration.MetricsServerConfig.Resources != nil {
+					d.Spec.Template.Spec.Containers[i].Resources = *f.config.ClusterMonitoringConfiguration.MetricsServerConfig.Resources
+				}
+
+				if f.config.ClusterMonitoringConfiguration.MetricsServerConfig.NodeSelector != nil {
+					d.Spec.Template.Spec.NodeSelector = f.config.ClusterMonitoringConfiguration.MetricsServerConfig.NodeSelector
+				}
+
+				if f.config.ClusterMonitoringConfiguration.MetricsServerConfig.Tolerations != nil {
+					d.Spec.Template.Spec.Tolerations = f.config.ClusterMonitoringConfiguration.MetricsServerConfig.Tolerations
+				}
+			}
+		}
+	}
+
+	return d, nil
+}
+
+func (f *Factory) MetricsServerPodDisruptionBudget() (*policyv1.PodDisruptionBudget, error) {
+	return f.NewPodDisruptionBudget(f.assets.MustNewAssetReader(MetricsServerPodDisruptionBudget))
+}
+
+func (f *Factory) MetricsServerService() (*v1.Service, error) {
+	return f.NewService(f.assets.MustNewAssetReader(MetricsServerService))
+}
+
+func (f *Factory) MetricsServerServiceMonitor() (*monv1.ServiceMonitor, error) {
+	return f.NewServiceMonitor(f.assets.MustNewAssetReader(MetricsServerServiceMonitor))
+}
+
+func (f *Factory) MetricsServerAPIService() (*apiregistrationv1.APIService, error) {
+	return f.NewAPIService(f.assets.MustNewAssetReader(MetricsServerAPIService))
+}
+
 func (f *Factory) PrometheusOperatorServiceMonitor() (*monv1.ServiceMonitor, error) {
 	return f.NewServiceMonitor(f.assets.MustNewAssetReader(PrometheusOperatorServiceMonitor))
 }

pkg/manifests/types.go

@@ -43,6 +43,8 @@ type ClusterMonitoringConfiguration struct {
 	HTTPConfig *HTTPConfig `json:"http,omitempty"`
 	// `K8sPrometheusAdapter` defines settings for the Prometheus Adapter component.
 	K8sPrometheusAdapter *K8sPrometheusAdapter `json:"k8sPrometheusAdapter,omitempty"`
+	// `MetricsServer` defines settings for the MetricsServer component.
+	MetricsServerConfig *MetricsServerConfig `json:"metricsServer,omitempty"`
 	// `KubeStateMetricsConfig` defines settings for the `kube-state-metrics` agent.
 	KubeStateMetricsConfig *KubeStateMetricsConfig `json:"kubeStateMetrics,omitempty"`
 	// `PrometheusK8sConfig` defines settings for the Prometheus component.
@@ -115,6 +117,18 @@ type K8sPrometheusAdapter struct {
 	DedicatedServiceMonitors *DedicatedServiceMonitors `json:"dedicatedServiceMonitors,omitempty"`
 }
 
+// The `MetricsServerConfig` resource defines settings for the MetricsServer component.
+type MetricsServerConfig struct {
+	// `Enabled` is a Boolean flag that enables or disables metrics-server.
+	Enabled *bool `json:"enabled,omitempty"`
+	// Defines the nodes on which the pods are scheduled.
+	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
+	// Defines tolerations for the pods.
+	Tolerations []v1.Toleration `json:"tolerations,omitempty"`
+	// Defines resource requests and limits for the Metrics Server container.
+	Resources *v1.ResourceRequirements `json:"resources,omitempty"`
+}
+
 // You can use the `DedicatedServiceMonitors` resource to configure dedicated
 // Service Monitors for the Prometheus Adapter
 type DedicatedServiceMonitors struct {

pkg/operator/operator.go

@@ -724,6 +724,7 @@ func (o *Operator) sync(ctx context.Context, key string) error {
 				tasks.NewTaskSpec("Updating kube-state-metrics", tasks.NewKubeStateMetricsTask(o.client, factory)),
 				tasks.NewTaskSpec("Updating openshift-state-metrics", tasks.NewOpenShiftStateMetricsTask(o.client, factory)),
 				tasks.NewTaskSpec("Updating prometheus-adapter", tasks.NewPrometheusAdapterTask(ctx, o.namespace, o.client, factory, config)),
+				tasks.NewTaskSpec("Updating metrics-server", tasks.NewMetricsServerTask(ctx, o.namespace, o.client, factory, config)),
 				tasks.NewTaskSpec("Updating Telemeter client", tasks.NewTelemeterClientTask(o.client, factory, config)),
 				tasks.NewTaskSpec("Updating Thanos Querier", tasks.NewThanosQuerierTask(o.client, factory, config)),
 				tasks.NewTaskSpec("Updating User Workload Thanos Ruler", tasks.NewThanosRulerUserWorkloadTask(o.client, factory, config)),