Merge branch 'main' into res_update

.gitignore

@@ -97,3 +97,4 @@ results.xml
 
 # build/tests artifacts
 registration-operator
+tmp

Makefile

@@ -7,6 +7,7 @@ include .bingo/Variables.mk
 FILES_TO_FMT ?= $(shell find . -path ./vendor -prune -o -name '*.deepcopy.go' -prune -o -name '*.go' -print)
 TMP_DIR := $(shell pwd)/tmp
 BIN_DIR ?= $(TMP_DIR)/bin
+export PATH := $(BIN_DIR):$(PATH)
 GIT ?= $(shell which git)
 
 XARGS ?= $(shell which gxargs 2>/dev/null || which xargs)
@@ -54,14 +55,15 @@ unit-tests-collectors:
 	go test ${VERBOSE} `go list ./collectors/... | $(GREP) -v test`
 
 .PHONY: e2e-tests
-e2e-tests:
+e2e-tests: install-e2e-test-deps
 	@echo "Running e2e tests ..."
 	@./cicd-scripts/run-e2e-tests.sh
 
 .PHONY: e2e-tests-in-kind
-e2e-tests-in-kind:
+e2e-tests-in-kind: install-e2e-test-deps
 	@echo "Running e2e tests in KinD cluster..."
 ifeq ($(OPENSHIFT_CI),true)
+    # Set up environment specific to OpenShift CI
 	@./cicd-scripts/run-e2e-in-kind-via-prow.sh
 else
 	@./tests/run-in-kind/run-e2e-in-kind.sh
@@ -151,3 +153,8 @@ io/ioutil.{Discard,NopCloser,ReadAll,ReadDir,ReadFile,TempDir,TempFile,Writefile
 .PHONY: install-build-deps
 install-build-deps:
 	@./scripts/install-binaries.sh install_build_deps
+
+.PHONY: install-e2e-test-deps
+install-e2e-test-deps:
+	@mkdir -p $(BIN_DIR)
+	@./scripts/install-binaries.sh install_e2e_tests_deps $(BIN_DIR)

cicd-scripts/customize-mco.sh

@@ -17,22 +17,8 @@ elif [[ "$(uname)" == "Darwin" ]]; then
   SED_COMMAND='sed -i '-e' -e'
 fi
 
-# Use snapshot for target release. Use latest one if no branch info detected, or not a release branch
-BRANCH=""
-LATEST_SNAPSHOT="2.10.0-SNAPSHOT-2024-02-22-17-32-31"
-if [[ ${PULL_BASE_REF} == "release-"* ]]; then
-  BRANCH=${PULL_BASE_REF#"release-"}
-  BRANCH=$(curl https://quay.io//api/v1/repository/stolostron/multicluster-observability-operator | jq '.tags|with_entries(select(.key|contains("'${BRANCH}'")))|keys[length-1]' | awk -F '-' '{print $1}')
-  BRANCH="${BRANCH#\"}"
-  LATEST_SNAPSHOT=$(curl https://quay.io//api/v1/repository/stolostron/multicluster-observability-operator | jq '.tags|with_entries(select(.key|contains("'${BRANCH}'-SNAPSHOT")))|keys[length-1]')
-fi
-if [[ ${LATEST_SNAPSHOT} == "null" ]] || [[ ${LATEST_SNAPSHOT} == "" ]]; then
-  LATEST_SNAPSHOT=$(curl https://quay.io/api/v1/repository/stolostron/multicluster-observability-operator | jq '.tags|with_entries(select((.key|contains("SNAPSHOT"))and(.key|contains("9.9.0")|not)))|keys[length-1]')
-fi
-
-# trim the leading and tailing quotes
-LATEST_SNAPSHOT="${LATEST_SNAPSHOT#\"}"
-LATEST_SNAPSHOT="${LATEST_SNAPSHOT%\"}"
+# Set the latest snapshot if it is not set
+LATEST_SNAPSHOT=${LATEST_SNAPSHOT:-$(get_latest_snapshot)}
 
 # list all components need to do test.
 CHANGED_COMPONENTS=""

cicd-scripts/run-e2e-in-kind-via-prow.sh

@@ -15,6 +15,9 @@ OPT=(-q -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" -i "${KE
 # support gnu sed only give that this script will be executed in prow env
 SED_COMMAND='sed -i-e -e'
 
+source ./scripts/test-utils.sh
+${SED_COMMAND} "$ a\export LATEST_SNAPSHOT=$(get_latest_snapshot)" ./tests/run-in-kind/env.sh
+
 if [ "${OPENSHIFT_CI}" == "true" ]; then
   ${SED_COMMAND} "$ a\export OPENSHIFT_CI=${OPENSHIFT_CI}" ./tests/run-in-kind/env.sh
 fi
@@ -40,5 +43,9 @@ if [[ -n ${RBAC_QUERY_PROXY_IMAGE_REF} ]]; then
 fi
 
 ssh "${OPT[@]}" "$HOST" sudo yum install gcc git -y
+ssh "${OPT[@]}" "$HOST" sudo mkdir -p /home/ec2-user/bin
+ssh "${OPT[@]}" "$HOST" sudo chmod 777 /home/ec2-user/bin
 scp "${OPT[@]}" -r ../multicluster-observability-operator "$HOST:/tmp/multicluster-observability-operator"
+scp "${OPT[@]}" $(which kubectl) "$HOST:/home/ec2-user/bin"
+scp "${OPT[@]}" $(which kustomize) "$HOST:/home/ec2-user/bin"
 ssh "${OPT[@]}" "$HOST" "cd /tmp/multicluster-observability-operator/tests/run-in-kind && ./run-e2e-in-kind.sh" > >(tee "$ARTIFACT_DIR/run-e2e-in-kind.log") 2>&1

cicd-scripts/setup-e2e-tests.sh

@@ -16,9 +16,6 @@ ROOTDIR="$(
   cd "$(dirname "$0")/.."
   pwd -P
 )"
-# Create bin directory and add it to PATH
-mkdir -p ${ROOTDIR}/bin
-export PATH=${PATH}:${ROOTDIR}/bin
 
 OCM_DEFAULT_NS="open-cluster-management"
 AGENT_NS="open-cluster-management-agent"
@@ -32,53 +29,8 @@ if [[ "$(uname)" == "Darwin" ]]; then
   SED_COMMAND='sed -i '-e' -e'
 fi
 
-# install jq
-if ! command -v jq &>/dev/null; then
-  if [[ "$(uname)" == "Linux" ]]; then
-    curl -o jq -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
-  elif [[ "$(uname)" == "Darwin" ]]; then
-    curl -o jq -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64
-  fi
-  chmod +x ./jq && mv ./jq ${ROOTDIR}/bin/jq
-fi
-
-# Use snapshot for target release. Use latest one if no branch info detected, or not a release branch
-BRANCH=""
-LATEST_SNAPSHOT=""
-if [[ ${PULL_BASE_REF} == "release-"* ]]; then
-  BRANCH=${PULL_BASE_REF#"release-"}
-  LATEST_SNAPSHOT=$(curl https://quay.io//api/v1/repository/open-cluster-management/multicluster-observability-operator | jq '.tags|with_entries(select(.key|test("'${BRANCH}'.*-SNAPSHOT-*")))|keys[length-1]')
-fi
-if [[ ${LATEST_SNAPSHOT} == "null" ]] || [[ ${LATEST_SNAPSHOT} == "" ]]; then
-  LATEST_SNAPSHOT=$(curl https://quay.io/api/v1/repository/stolostron/multicluster-observability-operator | jq '.tags|with_entries(select((.key|contains("SNAPSHOT"))and(.key|contains("9.9.0")|not)))|keys[length-1]')
-fi
-
-# trim the leading and tailing quotes
-LATEST_SNAPSHOT="${LATEST_SNAPSHOT#\"}"
-LATEST_SNAPSHOT="${LATEST_SNAPSHOT%\"}"
-
-# install kubectl
-if ! command -v kubectl &>/dev/null; then
-  echo "This script will install kubectl (https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your machine"
-  if [[ "$(uname)" == "Linux" ]]; then
-    curl -LO https://dl.k8s.io/release/v1.28.2/bin/linux/amd64/kubectl
-  elif [[ "$(uname)" == "Darwin" ]]; then
-    curl -LO curl -LO "https://dl.k8s.io/release/v1.28.2/bin/darwin/arm64/kubectl"
-  fi
-  chmod +x ./kubectl && mv ./kubectl ${ROOTDIR}/bin/kubectl
-fi
-
-# install kustomize
-if ! command -v kustomize &>/dev/null; then
-  echo "This script will install kustomize (sigs.k8s.io/kustomize/kustomize) on your machine"
-  if [[ "$(uname)" == "Linux" ]]; then
-    curl -o kustomize_v5.1.1.tar.gz -L https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.1.1/kustomize_v5.1.1_linux_amd64.tar.gz
-  elif [[ "$(uname)" == "Darwin" ]]; then
-    curl -o kustomize_v5.1.1.tar.gz -L https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.1.1/kustomize_v5.1.1_darwin_amd64.tar.gz
-  fi
-  tar xzvf kustomize_v5.1.1.tar.gz
-  chmod +x ./kustomize && mv ./kustomize ${ROOTDIR}/bin/kustomize
-fi
+# Set the latest snapshot if it is not set
+LATEST_SNAPSHOT=${LATEST_SNAPSHOT:-$(get_latest_snapshot)}
 
 # deploy the hub and spoke core via OLM
 deploy_hub_spoke_core() {

collectors/metrics/test/integration/setup.sh

@@ -22,41 +22,13 @@ if [[ "$(uname)" == "Darwin" ]]; then
 fi
 
 deploy() {
-  #setup_kubectl_command
   create_kind_hub
   deploy_prometheus_operator
   deploy_observatorium
   deploy_thanos
   deploy_metrics_collector $IMAGE_NAME
 }
 
-setup_kubectl_command() {
-  echo "=====Setup kubectl====="
-  # kubectl required for kind
-  echo "Install kubectl from openshift mirror (https://mirror.openshift.com/pub/openshift-v4/clients/ocp/4.4.14/openshift-client-mac-4.4.14.tar.gz)"
-  mv README.md README.md.tmp
-  if [[ "$(uname)" == "Darwin" ]]; then # then we are on a Mac
-    curl -LO https://mirror.openshift.com/pub/openshift-v4/clients/ocp/4.4.14/openshift-client-mac-4.4.14.tar.gz
-    tar xzvf openshift-client-mac-4.4.14.tar.gz # xzf to quiet logs
-    rm openshift-client-mac-4.4.14.tar.gz
-  elif [[ "$(uname)" == "Linux" ]]; then # we are in travis, building in rhel
-    curl -LO https://mirror.openshift.com/pub/openshift-v4/clients/ocp/4.4.14/openshift-client-linux-4.4.14.tar.gz
-    tar xzvf openshift-client-linux-4.4.14.tar.gz # xzf to quiet logs
-    rm openshift-client-linux-4.4.14.tar.gz
-  fi
-  # this package has a binary, so:
-
-  echo "Current directory"
-  echo $(pwd)
-  mv README.md.tmp README.md
-  chmod +x ./kubectl
-  if [[ ! -f /usr/local/bin/kubectl ]]; then
-    sudo cp ./kubectl /usr/local/bin/kubectl
-  fi
-  # kubectl are now installed in current dir
-  echo -n "kubectl version" && kubectl version
-}
-
 create_kind_hub() {
   WORKDIR=$(pwd)
   echo "Delete hub if it exists"

operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go

@@ -37,6 +37,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
@@ -236,7 +237,7 @@ func (r *MultiClusterObservabilityReconciler) Reconcile(ctx context.Context, req
 	//instance.Namespace = config.GetDefaultNamespace()
 	instance.Spec.StorageConfig.StorageClass = storageClassSelected
 	//Render the templates with a specified CR
-	renderer := rendering.NewMCORenderer(instance)
+	renderer := rendering.NewMCORenderer(instance, r.Client)
 	toDeploy, err := renderer.Render()
 	if err != nil {
 		reqLogger.Error(err, "Failed to render multiClusterMonitoring templates")
@@ -454,7 +455,20 @@ func (r *MultiClusterObservabilityReconciler) SetupWithManager(mgr ctrl.Manager)
 		Watches(&source.Kind{Type: &corev1.Secret{}}, &handler.EnqueueRequestForObject{}, builder.WithPredicates(secretPred)).
 		// Watch the namespace for changes
 		Watches(&source.Kind{Type: &corev1.Namespace{}}, &handler.EnqueueRequestForObject{},
-			builder.WithPredicates(namespacePred))
+			builder.WithPredicates(namespacePred)).
+		// Watch the kube-system extension-apiserver-authentication ConfigMap for changes
+		Watches(&source.Kind{Type: &corev1.ConfigMap{}}, handler.EnqueueRequestsFromMapFunc(
+			func(a client.Object) []reconcile.Request {
+				if a.GetName() == "extension-apiserver-authentication" && a.GetNamespace() == "kube-system" {
+					return []reconcile.Request{
+						{NamespacedName: types.NamespacedName{
+							Name:      "alertmanager-clientca-metric",
+							Namespace: config.GetMCONamespace(),
+						}},
+					}
+				}
+				return nil
+			}), builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}))
 
 	mchGroupKind := schema.GroupKind{Group: mchv1.GroupVersion.Group, Kind: "MultiClusterHub"}
 	if _, err := r.RESTMapper.RESTMapping(mchGroupKind, mchv1.GroupVersion.Version); err == nil {

operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller_test.go

@@ -108,6 +108,7 @@ var testImagemanifestsMap = map[string]string{
 	"rbac_query_proxy":             "test.io/rbac-query-proxy:test",
 	"thanos":                       "test.io/thanos:test",
 	"thanos_receive_controller":    "test.io/thanos_receive_controller:test",
+	"kube_rbac_proxy":              "test.io/kube-rbac-proxy:test",
 }
 
 func newTestImageManifestsConfigMap(namespace, version string) *corev1.ConfigMap {
@@ -317,10 +318,18 @@ func TestMultiClusterMonitoringCRUpdate(t *testing.T) {
 	testAmRouteBYOCaSecret := newTestCert(config.AlertmanagerRouteBYOCAName, namespace)
 	testAmRouteBYOCertSecret := newTestCert(config.AlertmanagerRouteBYOCERTName, namespace)
 	clustermgmtAddon := newClusterManagementAddon()
+	extensionApiserverAuthenticationCM := &corev1.ConfigMap{ // required by alertmanager
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "extension-apiserver-authentication",
+			Namespace: "kube-system",
+		},
+		Data: map[string]string{
+			"client-ca-file": "test",
+		},
+	}
 
 	objs := []runtime.Object{mco, svc, serverCACerts, clientCACerts, proxyRouteBYOCACerts, grafanaCert, serverCert,
-		testAmRouteBYOCaSecret, testAmRouteBYOCertSecret, proxyRouteBYOCert, clustermgmtAddon}
-
+		testAmRouteBYOCaSecret, testAmRouteBYOCertSecret, proxyRouteBYOCert, clustermgmtAddon, extensionApiserverAuthenticationCM}
 	// Create a fake client to mock API calls.
 	cl := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build()
 
@@ -713,9 +722,18 @@ func TestImageReplaceForMCO(t *testing.T) {
 	testAmRouteBYOCaSecret := newTestCert(config.AlertmanagerRouteBYOCAName, namespace)
 	testAmRouteBYOCertSecret := newTestCert(config.AlertmanagerRouteBYOCERTName, namespace)
 	clustermgmtAddon := newClusterManagementAddon()
+	extensionApiserverAuthenticationCM := &corev1.ConfigMap{ // required by alertmanager
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "extension-apiserver-authentication",
+			Namespace: "kube-system",
+		},
+		Data: map[string]string{
+			"client-ca-file": "test",
+		},
+	}
 
 	objs := []runtime.Object{mco, observatoriumAPIsvc, serverCACerts, clientCACerts, grafanaCert, serverCert,
-		testMCHInstance, imageManifestsCM, testAmRouteBYOCaSecret, testAmRouteBYOCertSecret, clustermgmtAddon}
+		testMCHInstance, imageManifestsCM, testAmRouteBYOCaSecret, testAmRouteBYOCertSecret, clustermgmtAddon, extensionApiserverAuthenticationCM}
 	// Create a fake client to mock API calls.
 	cl := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build()
 

operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-accessor-serviceaccount.yaml

@@ -2,6 +2,5 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: observability-alertmanager-accessor
-  namespace: open-cluster-management
   labels:
     alertmanager: observability

operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-cabundle.yaml

@@ -8,4 +8,3 @@ metadata:
   labels:
     alertmanager: observability
   name: alertmanager-ca-bundle
-  namespace: open-cluster-management

operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-clientca-metric.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+data: {}
+kind: ConfigMap
+metadata:
+  name: alertmanager-clientca-metric

operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-clusterrole.yaml

@@ -17,3 +17,12 @@ rules:
   - subjectaccessreviews
   verbs:
   - create
+- apiGroups: [""]
+  resources:
+  - configmaps
+  resourceNames:
+  - extension-apiserver-authentication
+  verbs:
+  - get
+  - watch
+

operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-config.yaml

@@ -19,7 +19,6 @@ stringData:
 kind: Secret
 metadata:
   name: alertmanager-config
-  namespace: open-cluster-management
   annotations:
     skip-creation-if-exist: "true"
 type: Opaque

operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-kube-rbac-proxy-metric.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: alertmanager-kube-rbac-proxy-metric
+stringData:
+  config.yaml: |-
+    "authorization":
+      "static":
+      - "path": "/metrics"
+        "resourceRequest": false
+        "user":
+          "name": "system:serviceaccount:openshift-monitoring:prometheus-k8s"
+        "verb": "get"    
+type: Opaque

operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-operated.yaml

@@ -2,14 +2,9 @@ apiVersion: v1
 kind: Service
 metadata:
   name: alertmanager-operated
-  namespace: open-cluster-management
 spec:
   clusterIP: None
   ports:
-  - name: web
-    port: 9093
-    protocol: TCP
-    targetPort: 9093
   - name: tcp-mesh
     port: 9094
     protocol: TCP

operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-proxy.yaml

@@ -5,7 +5,6 @@ metadata:
   labels:
     app.kubernetes.io/name: alertmanager-proxy
   name: alertmanager-proxy
-  namespace: open-cluster-management
   annotations:
     skip-creation-if-exist: "true"
 type: Opaque