Merge pull request #1178 from alvaroaleman/ff

[release-4.10] Forward from main

api/v1alpha1/hosted_controlplane.go

@@ -34,6 +34,7 @@ type HostedControlPlaneSpec struct {
 	// NetworkType specifies the SDN provider used for cluster networking.
 	NetworkType NetworkType                 `json:"networkType"`
 	SSHKey      corev1.LocalObjectReference `json:"sshKey"`
+	ClusterID   string                      `json:"clusterID"`
 	InfraID     string                      `json:"infraID"`
 	Platform    PlatformSpec                `json:"platform"`
 	DNS         DNSSpec                     `json:"dns"`

api/v1alpha1/hostedcluster_types.go

@@ -100,6 +100,19 @@ type HostedClusterSpec struct {
 	// and InfrastructureAvailabilityPolicy.
 	Release Release `json:"release"`
 
+	// ClusterID uniquely identifies this cluster. This is expected to be
+	// an RFC4122 UUID value (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx in
+	// hexadecimal values).
+	// As with a Kubernetes metadata.uid, this ID uniquely identifies this
+	// cluster in space and time.
+	// This value identifies the cluster in metrics pushed to telemetry and
+	// metrics produced by the control plane operators. If a value is not
+	// specified, an ID is generated. After initial creation, the value is
+	// immutable.
+	// +kubebuilder:validation:Pattern:="[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}"
+	// +optional
+	ClusterID string `json:"clusterID,omitempty"`
+
 	// InfraID is a globally unique identifier for the cluster. This identifier
 	// will be used to associate various cloud resources with the HostedCluster
 	// and its associated NodePools.

cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedclusters.yaml

@@ -116,6 +116,16 @@ spec:
                     format: int32
                     type: integer
                 type: object
+              clusterID:
+                description: ClusterID uniquely identifies this cluster. This is expected
+                  to be an RFC4122 UUID value (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+                  in hexadecimal values). As with a Kubernetes metadata.uid, this
+                  ID uniquely identifies this cluster in space and time. This value
+                  identifies the cluster in metrics pushed to telemetry and metrics
+                  produced by the control plane operators. If a value is not specified,
+                  an ID is generated. After initial creation, the value is immutable.
+                pattern: '[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}'
+                type: string
               configuration:
                 description: Configuration specifies configuration for individual
                   OCP components in the cluster, represented as embedded resources

cmd/install/assets/hypershift-operator/hypershift.openshift.io_hostedcontrolplanes.yaml

@@ -65,6 +65,8 @@ spec:
                       TODO: Add other useful fields. apiVersion, kind, uid?'
                     type: string
                 type: object
+              clusterID:
+                type: string
               configuration:
                 description: 'Configuration embeds resources that correspond to the
                   openshift configuration API: https://docs.openshift.com/container-platform/4.7/rest_api/config_apis/config-apis-index.html'
@@ -861,6 +863,7 @@ spec:
                     type: string
                 type: object
             required:
+            - clusterID
             - dns
             - etcd
             - infraID

cmd/install/assets/hypershift_operator.go

@@ -740,11 +740,21 @@ func (o HyperShiftOperatorClusterRole) Build() *rbacv1.ClusterRole {
 				Resources: []string{"virtualmachineinstances", "virtualmachines"},
 				Verbs:     []string{"*"},
 			},
-			{
+			{ // This allows hypershift operator to grant RBAC permissions for agents, clusterDeployments and agentClusterInstalls to the capi-provider-agent
 				APIGroups: []string{"agent-install.openshift.io"},
 				Resources: []string{"agents"},
 				Verbs:     []string{"*"},
 			},
+			{
+				APIGroups: []string{"extensions.hive.openshift.io"},
+				Resources: []string{"agentclusterinstalls"},
+				Verbs:     []string{"*"},
+			},
+			{
+				APIGroups: []string{"hive.openshift.io"},
+				Resources: []string{"clusterdeployments"},
+				Verbs:     []string{"*"},
+			},
 		},
 	}
 	return role

control-plane-operator/controllers/hostedcontrolplane/cvo/params.go

@@ -7,20 +7,25 @@ import (
 	hyperv1 "github.com/openshift/hypershift/api/v1alpha1"
 
 	"github.com/openshift/hypershift/support/config"
+	"github.com/openshift/hypershift/support/util"
 )
 
 type CVOParams struct {
-	Image            string
-	CLIImage         string
-	OwnerRef         config.OwnerRef
-	DeploymentConfig config.DeploymentConfig
+	Image                   string
+	CLIImage                string
+	AvailabilityProberImage string
+	ClusterID               string
+	OwnerRef                config.OwnerRef
+	DeploymentConfig        config.DeploymentConfig
 }
 
 func NewCVOParams(hcp *hyperv1.HostedControlPlane, images map[string]string, setDefaultSecurityContext bool) *CVOParams {
 	p := &CVOParams{
-		CLIImage: images["cli"],
-		Image:    hcp.Spec.ReleaseImage,
-		OwnerRef: config.OwnerRefFrom(hcp),
+		CLIImage:                images["cli"],
+		AvailabilityProberImage: images[util.AvailabilityProberImageName],
+		Image:                   hcp.Spec.ReleaseImage,
+		OwnerRef:                config.OwnerRefFrom(hcp),
+		ClusterID:               hcp.Spec.ClusterID,
 	}
 	p.DeploymentConfig.Resources = config.ResourcesSpec{
 		cvoContainerPrepPayload().Name: {

control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go

@@ -31,7 +31,7 @@ var (
 		cvoContainerPrepPayload().Name: {
 			cvoVolumePayload().Name: "/var/payload",
 		},
-		cvoContainerApplyBootstrap().Name: {
+		cvoContainerBootstrap().Name: {
 			cvoVolumePayload().Name:    "/var/payload",
 			cvoVolumeKubeconfig().Name: "/etc/kubernetes",
 		},
@@ -83,7 +83,7 @@ func cvoLabels() map[string]string {
 
 var port int32 = 8443
 
-func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef, deploymentConfig config.DeploymentConfig, image, cliImage string) error {
+func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef, deploymentConfig config.DeploymentConfig, image, cliImage, availabilityProberImage, clusterID string, apiPort *int32) error {
 	ownerRef.ApplyTo(deployment)
 
 	// preserve existing resource requirements for main CVO container
@@ -104,10 +104,10 @@ func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef
 				AutomountServiceAccountToken: pointer.BoolPtr(false),
 				InitContainers: []corev1.Container{
 					util.BuildContainer(cvoContainerPrepPayload(), buildCVOContainerPrepPayload(image)),
+					util.BuildContainer(cvoContainerBootstrap(), buildCVOContainerBootstrap(cliImage, clusterID)),
 				},
 				Containers: []corev1.Container{
 					util.BuildContainer(cvoContainerMain(), buildCVOContainerMain(image)),
-					util.BuildContainer(cvoContainerApplyBootstrap(), buildCVOContainerApplyBootstrap(cliImage)),
 				},
 				Volumes: []corev1.Volume{
 					util.BuildVolume(cvoVolumePayload(), buildCVOVolumePayload),
@@ -119,6 +119,14 @@ func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef
 		},
 	}
 	deploymentConfig.ApplyTo(deployment)
+	util.AvailabilityProber(
+		kas.InClusterKASReadyURL(deployment.Namespace, apiPort),
+		availabilityProberImage,
+		&deployment.Spec.Template.Spec,
+		func(o *util.AvailabilityProberOpts) {
+			o.KubeconfigVolumeName = cvoVolumeKubeconfig().Name
+		},
+	)
 	return nil
 }
 
@@ -128,9 +136,9 @@ func cvoContainerPrepPayload() *corev1.Container {
 	}
 }
 
-func cvoContainerApplyBootstrap() *corev1.Container {
+func cvoContainerBootstrap() *corev1.Container {
 	return &corev1.Container{
-		Name: "apply-bootstrap",
+		Name: "bootstrap",
 	}
 }
 
@@ -152,13 +160,13 @@ func buildCVOContainerPrepPayload(image string) func(c *corev1.Container) {
 	}
 }
 
-func buildCVOContainerApplyBootstrap(image string) func(*corev1.Container) {
+func buildCVOContainerBootstrap(image, clusterID string) func(*corev1.Container) {
 	return func(c *corev1.Container) {
 		c.Image = image
 		c.Command = []string{"/bin/bash"}
 		c.Args = []string{
 			"-c",
-			applyBootrapScript(),
+			cvoBootrapScript(clusterID),
 		}
 		c.Resources.Requests = corev1.ResourceList{
 			corev1.ResourceCPU:    resource.MustParse("10m"),
@@ -189,10 +197,20 @@ func preparePayloadScript() string {
 	return strings.Join(stmts, "\n")
 }
 
-func applyBootrapScript() string {
-	payloadDir := volumeMounts.Path(cvoContainerApplyBootstrap().Name, cvoVolumePayload().Name)
-	var script = `#!/bin/bash
+func cvoBootrapScript(clusterID string) string {
+	payloadDir := volumeMounts.Path(cvoContainerBootstrap().Name, cvoVolumePayload().Name)
+	var scriptTemplate = `#!/bin/bash
 set -euo pipefail
+cat > /tmp/clusterversion.yaml <<EOF
+apiVersion: config.openshift.io/v1
+kind: ClusterVersion
+metadata:
+  name: version
+spec:
+  clusterID: %s
+EOF
+oc get ns openshift-config &> /dev/null || oc create ns openshift-config
+oc get ns openshift-config-managed &> /dev/null || oc create ns openshift-config-managed
 while true; do
   echo "Applying CVO bootstrap manifests"
   if oc apply -f %s/manifests; then
@@ -201,16 +219,9 @@ while true; do
   fi
   sleep 1
 done
-
-# After bootstrap manifests have been applied, just sleep to keep the container alive.
-# The reason the container must be run in parallel (and not as an init container) is that
-# some manifests in the bootstrap directory depend on manifests from the release payload
-# that is applied by the main CVO container.
-while true; do
-  sleep 1000
-done
+oc get clusterversion/version &> /dev/null || oc create -f /tmp/clusterversion.yaml
 `
-	return fmt.Sprintf(script, payloadDir)
+	return fmt.Sprintf(scriptTemplate, clusterID, payloadDir)
 }
 
 func buildCVOContainerMain(image string) func(c *corev1.Container) {
@@ -225,7 +236,6 @@ func buildCVOContainerMain(image string) func(c *corev1.Container) {
 			"--release-image",
 			image,
 			"--enable-auto-update=false",
-			"--enable-default-cluster-version=true",
 			"--kubeconfig",
 			path.Join(volumeMounts.Path(c.Name, cvoVolumeKubeconfig().Name), kas.KubeconfigKey),
 			fmt.Sprintf("--listen=0.0.0.0:%d", port),
@@ -336,7 +346,7 @@ func ReconcileService(svc *corev1.Service, owner config.OwnerRef) error {
 	return nil
 }
 
-func ReconcileServiceMonitor(sm *prometheusoperatorv1.ServiceMonitor, ownerRef config.OwnerRef) error {
+func ReconcileServiceMonitor(sm *prometheusoperatorv1.ServiceMonitor, ownerRef config.OwnerRef, clusterID string) error {
 	ownerRef.ApplyTo(sm)
 
 	sm.Spec.Selector.MatchLabels = cvoLabels()
@@ -386,5 +396,7 @@ func ReconcileServiceMonitor(sm *prometheusoperatorv1.ServiceMonitor, ownerRef c
 		},
 	}
 
+	util.ApplyClusterIDLabel(&sm.Spec.Endpoints[0], clusterID)
+
 	return nil
 }

control-plane-operator/controllers/hostedcontrolplane/etcd/reconcile.go

@@ -253,7 +253,7 @@ func ReconcileClientService(service *corev1.Service, ownerRef config.OwnerRef) e
 // ReconcileServiceMonitor
 // TODO: Exposing the client cert to monitoring isn't great, but metrics
 // TLS can't yet be independently configured. See: https://github.com/etcd-io/etcd/pull/10504
-func ReconcileServiceMonitor(sm *prometheusoperatorv1.ServiceMonitor, ownerRef config.OwnerRef) error {
+func ReconcileServiceMonitor(sm *prometheusoperatorv1.ServiceMonitor, ownerRef config.OwnerRef, clusterID string) error {
 	ownerRef.ApplyTo(sm)
 
 	sm.Spec.Selector.MatchLabels = etcdPodSelector()
@@ -295,6 +295,8 @@ func ReconcileServiceMonitor(sm *prometheusoperatorv1.ServiceMonitor, ownerRef c
 		},
 	}
 
+	util.ApplyClusterIDLabel(&sm.Spec.Endpoints[0], clusterID)
+
 	return nil
 }