diff --git a/.gitignore b/.gitignore index e6e9e203e0..39364e4eca 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,4 @@ results.xml # build/tests artifacts registration-operator +tmp diff --git a/Makefile b/Makefile index e40ed03d4d..865fc3e6d2 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ include .bingo/Variables.mk FILES_TO_FMT ?= $(shell find . -path ./vendor -prune -o -name '*.deepcopy.go' -prune -o -name '*.go' -print) TMP_DIR := $(shell pwd)/tmp BIN_DIR ?= $(TMP_DIR)/bin +export PATH := $(BIN_DIR):$(PATH) GIT ?= $(shell which git) XARGS ?= $(shell which gxargs 2>/dev/null || which xargs) @@ -54,14 +55,15 @@ unit-tests-collectors: go test ${VERBOSE} `go list ./collectors/... | $(GREP) -v test` .PHONY: e2e-tests -e2e-tests: +e2e-tests: install-e2e-test-deps @echo "Running e2e tests ..." @./cicd-scripts/run-e2e-tests.sh .PHONY: e2e-tests-in-kind -e2e-tests-in-kind: +e2e-tests-in-kind: install-e2e-test-deps @echo "Running e2e tests in KinD cluster..." ifeq ($(OPENSHIFT_CI),true) + # Set up environment specific to OpenShift CI @./cicd-scripts/run-e2e-in-kind-via-prow.sh else @./tests/run-in-kind/run-e2e-in-kind.sh @@ -151,3 +153,8 @@ io/ioutil.{Discard,NopCloser,ReadAll,ReadDir,ReadFile,TempDir,TempFile,Writefile .PHONY: install-build-deps install-build-deps: @./scripts/install-binaries.sh install_build_deps + +.PHONY: install-e2e-test-deps +install-e2e-test-deps: + @mkdir -p $(BIN_DIR) + @./scripts/install-binaries.sh install_e2e_tests_deps $(BIN_DIR) diff --git a/cicd-scripts/build.sh b/cicd-scripts/build.sh deleted file mode 100755 index 16e7eee9bc..0000000000 --- a/cicd-scripts/build.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# Copyright (c) 2021 Red Hat, Inc. -# Copyright Contributors to the Open Cluster Management project - -set -e - -make docker-binary - -git config --global url."https://$GITHUB_TOKEN@github.com/stolostron".insteadOf "https://github.com/stolostron" - -echo "Building multicluster-observability-operator image" -export DOCKER_IMAGE_AND_TAG=${1} -export DOCKER_FILE=Dockerfile -make docker/build diff --git a/cicd-scripts/customize-mco.sh b/cicd-scripts/customize-mco.sh index f305e34148..ae7505e748 100755 --- a/cicd-scripts/customize-mco.sh +++ b/cicd-scripts/customize-mco.sh @@ -17,22 +17,8 @@ elif [[ "$(uname)" == "Darwin" ]]; then SED_COMMAND='sed -i '-e' -e' fi -# Use snapshot for target release. Use latest one if no branch info detected, or not a release branch -BRANCH="" -LATEST_SNAPSHOT="2.10.0-SNAPSHOT-2024-02-22-17-32-31" -if [[ ${PULL_BASE_REF} == "release-"* ]]; then - BRANCH=${PULL_BASE_REF#"release-"} - BRANCH=$(curl https://quay.io//api/v1/repository/stolostron/multicluster-observability-operator | jq '.tags|with_entries(select(.key|contains("'${BRANCH}'")))|keys[length-1]' | awk -F '-' '{print $1}') - BRANCH="${BRANCH#\"}" - LATEST_SNAPSHOT=$(curl https://quay.io//api/v1/repository/stolostron/multicluster-observability-operator | jq '.tags|with_entries(select(.key|contains("'${BRANCH}'-SNAPSHOT")))|keys[length-1]') -fi -if [[ ${LATEST_SNAPSHOT} == "null" ]] || [[ ${LATEST_SNAPSHOT} == "" ]]; then - LATEST_SNAPSHOT=$(curl https://quay.io/api/v1/repository/stolostron/multicluster-observability-operator | jq '.tags|with_entries(select((.key|contains("SNAPSHOT"))and(.key|contains("9.9.0")|not)))|keys[length-1]') -fi - -# trim the leading and tailing quotes -LATEST_SNAPSHOT="${LATEST_SNAPSHOT#\"}" -LATEST_SNAPSHOT="${LATEST_SNAPSHOT%\"}" +# Set the latest snapshot if it is not set +LATEST_SNAPSHOT=${LATEST_SNAPSHOT:-$(get_latest_snapshot)} # list all components need to do test. CHANGED_COMPONENTS="" diff --git a/cicd-scripts/run-e2e-in-kind-via-prow.sh b/cicd-scripts/run-e2e-in-kind-via-prow.sh index 9e3b4d9d2b..b882aac991 100755 --- a/cicd-scripts/run-e2e-in-kind-via-prow.sh +++ b/cicd-scripts/run-e2e-in-kind-via-prow.sh @@ -15,6 +15,9 @@ OPT=(-q -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" -i "${KE # support gnu sed only give that this script will be executed in prow env SED_COMMAND='sed -i-e -e' +source ./scripts/test-utils.sh +${SED_COMMAND} "$ a\export LATEST_SNAPSHOT=$(get_latest_snapshot)" ./tests/run-in-kind/env.sh + if [ "${OPENSHIFT_CI}" == "true" ]; then ${SED_COMMAND} "$ a\export OPENSHIFT_CI=${OPENSHIFT_CI}" ./tests/run-in-kind/env.sh fi @@ -40,5 +43,9 @@ if [[ -n ${RBAC_QUERY_PROXY_IMAGE_REF} ]]; then fi ssh "${OPT[@]}" "$HOST" sudo yum install gcc git -y +ssh "${OPT[@]}" "$HOST" sudo mkdir -p /home/ec2-user/bin +ssh "${OPT[@]}" "$HOST" sudo chmod 777 /home/ec2-user/bin scp "${OPT[@]}" -r ../multicluster-observability-operator "$HOST:/tmp/multicluster-observability-operator" +scp "${OPT[@]}" $(which kubectl) "$HOST:/home/ec2-user/bin" +scp "${OPT[@]}" $(which kustomize) "$HOST:/home/ec2-user/bin" ssh "${OPT[@]}" "$HOST" "cd /tmp/multicluster-observability-operator/tests/run-in-kind && ./run-e2e-in-kind.sh" > >(tee "$ARTIFACT_DIR/run-e2e-in-kind.log") 2>&1 diff --git a/cicd-scripts/run-unit-tests.sh b/cicd-scripts/run-unit-tests.sh deleted file mode 100755 index cfa21ba919..0000000000 --- a/cicd-scripts/run-unit-tests.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# Copyright (c) 2021 Red Hat, Inc. -# Copyright Contributors to the Open Cluster Management project - -echo "/: : $1" - -git config --global url."https://$GITHUB_TOKEN@github.com/stolostron".insteadOf "https://github.com/stolostron" - -go test ./... diff --git a/cicd-scripts/setup-e2e-tests.sh b/cicd-scripts/setup-e2e-tests.sh index 1573ba7d77..27039bddff 100755 --- a/cicd-scripts/setup-e2e-tests.sh +++ b/cicd-scripts/setup-e2e-tests.sh @@ -16,9 +16,6 @@ ROOTDIR="$( cd "$(dirname "$0")/.." pwd -P )" -# Create bin directory and add it to PATH -mkdir -p ${ROOTDIR}/bin -export PATH=${PATH}:${ROOTDIR}/bin OCM_DEFAULT_NS="open-cluster-management" AGENT_NS="open-cluster-management-agent" @@ -32,53 +29,8 @@ if [[ "$(uname)" == "Darwin" ]]; then SED_COMMAND='sed -i '-e' -e' fi -# install jq -if ! command -v jq &>/dev/null; then - if [[ "$(uname)" == "Linux" ]]; then - curl -o jq -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 - elif [[ "$(uname)" == "Darwin" ]]; then - curl -o jq -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 - fi - chmod +x ./jq && mv ./jq ${ROOTDIR}/bin/jq -fi - -# Use snapshot for target release. Use latest one if no branch info detected, or not a release branch -BRANCH="" -LATEST_SNAPSHOT="" -if [[ ${PULL_BASE_REF} == "release-"* ]]; then - BRANCH=${PULL_BASE_REF#"release-"} - LATEST_SNAPSHOT=$(curl https://quay.io//api/v1/repository/open-cluster-management/multicluster-observability-operator | jq '.tags|with_entries(select(.key|test("'${BRANCH}'.*-SNAPSHOT-*")))|keys[length-1]') -fi -if [[ ${LATEST_SNAPSHOT} == "null" ]] || [[ ${LATEST_SNAPSHOT} == "" ]]; then - LATEST_SNAPSHOT=$(curl https://quay.io/api/v1/repository/stolostron/multicluster-observability-operator | jq '.tags|with_entries(select((.key|contains("SNAPSHOT"))and(.key|contains("9.9.0")|not)))|keys[length-1]') -fi - -# trim the leading and tailing quotes -LATEST_SNAPSHOT="${LATEST_SNAPSHOT#\"}" -LATEST_SNAPSHOT="${LATEST_SNAPSHOT%\"}" - -# install kubectl -if ! command -v kubectl &>/dev/null; then - echo "This script will install kubectl (https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your machine" - if [[ "$(uname)" == "Linux" ]]; then - curl -LO https://dl.k8s.io/release/v1.28.2/bin/linux/amd64/kubectl - elif [[ "$(uname)" == "Darwin" ]]; then - curl -LO curl -LO "https://dl.k8s.io/release/v1.28.2/bin/darwin/arm64/kubectl" - fi - chmod +x ./kubectl && mv ./kubectl ${ROOTDIR}/bin/kubectl -fi - -# install kustomize -if ! command -v kustomize &>/dev/null; then - echo "This script will install kustomize (sigs.k8s.io/kustomize/kustomize) on your machine" - if [[ "$(uname)" == "Linux" ]]; then - curl -o kustomize_v5.1.1.tar.gz -L https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.1.1/kustomize_v5.1.1_linux_amd64.tar.gz - elif [[ "$(uname)" == "Darwin" ]]; then - curl -o kustomize_v5.1.1.tar.gz -L https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.1.1/kustomize_v5.1.1_darwin_amd64.tar.gz - fi - tar xzvf kustomize_v5.1.1.tar.gz - chmod +x ./kustomize && mv ./kustomize ${ROOTDIR}/bin/kustomize -fi +# Set the latest snapshot if it is not set +LATEST_SNAPSHOT=${LATEST_SNAPSHOT:-$(get_latest_snapshot)} # deploy the hub and spoke core via OLM deploy_hub_spoke_core() { diff --git a/collectors/metrics/test/integration/setup.sh b/collectors/metrics/test/integration/setup.sh index 521f50be35..ac155f9472 100755 --- a/collectors/metrics/test/integration/setup.sh +++ b/collectors/metrics/test/integration/setup.sh @@ -22,7 +22,6 @@ if [[ "$(uname)" == "Darwin" ]]; then fi deploy() { - #setup_kubectl_command create_kind_hub deploy_prometheus_operator deploy_observatorium @@ -30,33 +29,6 @@ deploy() { deploy_metrics_collector $IMAGE_NAME } -setup_kubectl_command() { - echo "=====Setup kubectl=====" - # kubectl required for kind - echo "Install kubectl from openshift mirror (https://mirror.openshift.com/pub/openshift-v4/clients/ocp/4.4.14/openshift-client-mac-4.4.14.tar.gz)" - mv README.md README.md.tmp - if [[ "$(uname)" == "Darwin" ]]; then # then we are on a Mac - curl -LO https://mirror.openshift.com/pub/openshift-v4/clients/ocp/4.4.14/openshift-client-mac-4.4.14.tar.gz - tar xzvf openshift-client-mac-4.4.14.tar.gz # xzf to quiet logs - rm openshift-client-mac-4.4.14.tar.gz - elif [[ "$(uname)" == "Linux" ]]; then # we are in travis, building in rhel - curl -LO https://mirror.openshift.com/pub/openshift-v4/clients/ocp/4.4.14/openshift-client-linux-4.4.14.tar.gz - tar xzvf openshift-client-linux-4.4.14.tar.gz # xzf to quiet logs - rm openshift-client-linux-4.4.14.tar.gz - fi - # this package has a binary, so: - - echo "Current directory" - echo $(pwd) - mv README.md.tmp README.md - chmod +x ./kubectl - if [[ ! -f /usr/local/bin/kubectl ]]; then - sudo cp ./kubectl /usr/local/bin/kubectl - fi - # kubectl are now installed in current dir - echo -n "kubectl version" && kubectl version -} - create_kind_hub() { WORKDIR=$(pwd) echo "Delete hub if it exists" diff --git a/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go b/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go index 6259a5b36a..2712d83545 100644 --- a/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go +++ b/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go @@ -37,6 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" @@ -236,7 +237,7 @@ func (r *MultiClusterObservabilityReconciler) Reconcile(ctx context.Context, req //instance.Namespace = config.GetDefaultNamespace() instance.Spec.StorageConfig.StorageClass = storageClassSelected //Render the templates with a specified CR - renderer := rendering.NewMCORenderer(instance) + renderer := rendering.NewMCORenderer(instance, r.Client) toDeploy, err := renderer.Render() if err != nil { reqLogger.Error(err, "Failed to render multiClusterMonitoring templates") @@ -454,7 +455,20 @@ func (r *MultiClusterObservabilityReconciler) SetupWithManager(mgr ctrl.Manager) Watches(&source.Kind{Type: &corev1.Secret{}}, &handler.EnqueueRequestForObject{}, builder.WithPredicates(secretPred)). // Watch the namespace for changes Watches(&source.Kind{Type: &corev1.Namespace{}}, &handler.EnqueueRequestForObject{}, - builder.WithPredicates(namespacePred)) + builder.WithPredicates(namespacePred)). + // Watch the kube-system extension-apiserver-authentication ConfigMap for changes + Watches(&source.Kind{Type: &corev1.ConfigMap{}}, handler.EnqueueRequestsFromMapFunc( + func(a client.Object) []reconcile.Request { + if a.GetName() == "extension-apiserver-authentication" && a.GetNamespace() == "kube-system" { + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Name: "alertmanager-clientca-metric", + Namespace: config.GetMCONamespace(), + }}, + } + } + return nil + }), builder.WithPredicates(predicate.ResourceVersionChangedPredicate{})) mchGroupKind := schema.GroupKind{Group: mchv1.GroupVersion.Group, Kind: "MultiClusterHub"} if _, err := r.RESTMapper.RESTMapping(mchGroupKind, mchv1.GroupVersion.Version); err == nil { diff --git a/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller_test.go b/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller_test.go index 6b49103cc4..9d4726d92d 100644 --- a/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller_test.go +++ b/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller_test.go @@ -108,6 +108,7 @@ var testImagemanifestsMap = map[string]string{ "rbac_query_proxy": "test.io/rbac-query-proxy:test", "thanos": "test.io/thanos:test", "thanos_receive_controller": "test.io/thanos_receive_controller:test", + "kube_rbac_proxy": "test.io/kube-rbac-proxy:test", } func newTestImageManifestsConfigMap(namespace, version string) *corev1.ConfigMap { @@ -317,10 +318,18 @@ func TestMultiClusterMonitoringCRUpdate(t *testing.T) { testAmRouteBYOCaSecret := newTestCert(config.AlertmanagerRouteBYOCAName, namespace) testAmRouteBYOCertSecret := newTestCert(config.AlertmanagerRouteBYOCERTName, namespace) clustermgmtAddon := newClusterManagementAddon() + extensionApiserverAuthenticationCM := &corev1.ConfigMap{ // required by alertmanager + ObjectMeta: metav1.ObjectMeta{ + Name: "extension-apiserver-authentication", + Namespace: "kube-system", + }, + Data: map[string]string{ + "client-ca-file": "test", + }, + } objs := []runtime.Object{mco, svc, serverCACerts, clientCACerts, proxyRouteBYOCACerts, grafanaCert, serverCert, - testAmRouteBYOCaSecret, testAmRouteBYOCertSecret, proxyRouteBYOCert, clustermgmtAddon} - + testAmRouteBYOCaSecret, testAmRouteBYOCertSecret, proxyRouteBYOCert, clustermgmtAddon, extensionApiserverAuthenticationCM} // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() @@ -713,9 +722,18 @@ func TestImageReplaceForMCO(t *testing.T) { testAmRouteBYOCaSecret := newTestCert(config.AlertmanagerRouteBYOCAName, namespace) testAmRouteBYOCertSecret := newTestCert(config.AlertmanagerRouteBYOCERTName, namespace) clustermgmtAddon := newClusterManagementAddon() + extensionApiserverAuthenticationCM := &corev1.ConfigMap{ // required by alertmanager + ObjectMeta: metav1.ObjectMeta{ + Name: "extension-apiserver-authentication", + Namespace: "kube-system", + }, + Data: map[string]string{ + "client-ca-file": "test", + }, + } objs := []runtime.Object{mco, observatoriumAPIsvc, serverCACerts, clientCACerts, grafanaCert, serverCert, - testMCHInstance, imageManifestsCM, testAmRouteBYOCaSecret, testAmRouteBYOCertSecret, clustermgmtAddon} + testMCHInstance, imageManifestsCM, testAmRouteBYOCaSecret, testAmRouteBYOCertSecret, clustermgmtAddon, extensionApiserverAuthenticationCM} // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() diff --git a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-accessor-serviceaccount.yaml b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-accessor-serviceaccount.yaml index 2dc9385dc5..854df0526c 100644 --- a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-accessor-serviceaccount.yaml +++ b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-accessor-serviceaccount.yaml @@ -2,6 +2,5 @@ apiVersion: v1 kind: ServiceAccount metadata: name: observability-alertmanager-accessor - namespace: open-cluster-management labels: alertmanager: observability diff --git a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-cabundle.yaml b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-cabundle.yaml index 9109c6c564..db0b6b5d96 100644 --- a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-cabundle.yaml +++ b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-cabundle.yaml @@ -8,4 +8,3 @@ metadata: labels: alertmanager: observability name: alertmanager-ca-bundle - namespace: open-cluster-management diff --git a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-clientca-metric.yaml b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-clientca-metric.yaml new file mode 100644 index 0000000000..b0f2a65867 --- /dev/null +++ b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-clientca-metric.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +data: {} +kind: ConfigMap +metadata: + name: alertmanager-clientca-metric \ No newline at end of file diff --git a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-clusterrole.yaml b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-clusterrole.yaml index aba7ea1a18..c9627e3b0c 100644 --- a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-clusterrole.yaml +++ b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-clusterrole.yaml @@ -17,3 +17,12 @@ rules: - subjectaccessreviews verbs: - create +- apiGroups: [""] + resources: + - configmaps + resourceNames: + - extension-apiserver-authentication + verbs: + - get + - watch + diff --git a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-config.yaml b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-config.yaml index 9bfabcd750..5aa2dec2e9 100644 --- a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-config.yaml +++ b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-config.yaml @@ -19,7 +19,6 @@ stringData: kind: Secret metadata: name: alertmanager-config - namespace: open-cluster-management annotations: skip-creation-if-exist: "true" type: Opaque \ No newline at end of file diff --git a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-kube-rbac-proxy-metric.yaml b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-kube-rbac-proxy-metric.yaml new file mode 100644 index 0000000000..87017cb4ea --- /dev/null +++ b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-kube-rbac-proxy-metric.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Secret +metadata: + name: alertmanager-kube-rbac-proxy-metric +stringData: + config.yaml: |- + "authorization": + "static": + - "path": "/metrics" + "resourceRequest": false + "user": + "name": "system:serviceaccount:openshift-monitoring:prometheus-k8s" + "verb": "get" +type: Opaque \ No newline at end of file diff --git a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-operated.yaml b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-operated.yaml index 5064614ddf..9862fd0de0 100644 --- a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-operated.yaml +++ b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-operated.yaml @@ -2,14 +2,9 @@ apiVersion: v1 kind: Service metadata: name: alertmanager-operated - namespace: open-cluster-management spec: clusterIP: None ports: - - name: web - port: 9093 - protocol: TCP - targetPort: 9093 - name: tcp-mesh port: 9094 protocol: TCP diff --git a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-proxy.yaml b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-proxy.yaml index 408fea83f1..fa9b7a4e0a 100644 --- a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-proxy.yaml +++ b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-proxy.yaml @@ -5,7 +5,6 @@ metadata: labels: app.kubernetes.io/name: alertmanager-proxy name: alertmanager-proxy - namespace: open-cluster-management annotations: skip-creation-if-exist: "true" type: Opaque diff --git a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-service-metrics.yaml b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-service-metrics.yaml new file mode 100644 index 0000000000..15d5e52b46 --- /dev/null +++ b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-service-metrics.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + # This annotation tells the service CA operator to provision a Secret + # holding the certificate + key to be mounted in the pods. + # The Secret name is "" (e.g. "secret-my-app-tls"). + service.beta.openshift.io/serving-cert-secret-name: alertmanager-tls-metrics + labels: + app: multicluster-observability-alertmanager-metrics + name: alertmanager-metrics +spec: + ports: + - name: metrics + port: 9096 + targetPort: metrics + selector: + alertmanager: observability + app: multicluster-observability-alertmanager + type: ClusterIP \ No newline at end of file diff --git a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-service.yaml b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-service.yaml index 29e5e498ae..3a0ca7d6b3 100644 --- a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-service.yaml +++ b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-service.yaml @@ -4,15 +4,10 @@ metadata: labels: alertmanager: observability name: alertmanager - namespace: open-cluster-management annotations: service.beta.openshift.io/serving-cert-secret-name: alertmanager-tls spec: ports: - - name: web - port: 9093 - protocol: TCP - targetPort: web - name: oauth-proxy port: 9095 protocol: TCP diff --git a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-serviceaccount.yaml b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-serviceaccount.yaml index 3de3467f14..551cdb9dde 100644 --- a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-serviceaccount.yaml +++ b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-serviceaccount.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: alertmanager - namespace: open-cluster-management labels: alertmanager: observability annotations: diff --git a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-servicemonitor.yaml b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-servicemonitor.yaml new file mode 100644 index 0000000000..4b1dad6c67 --- /dev/null +++ b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-servicemonitor.yaml @@ -0,0 +1,26 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: alertmanager +spec: + endpoints: + - interval: 30s + # Matches the name of the service's port. + port: metrics + scheme: https + tlsConfig: + # The name of the server (CN) in the server's certificate. + serverName: alertmanager-metrics.open-cluster-management-observability.svc + # The CA file used by Prometheus to verify the server's certificate. + # It's the cluster's CA bundle from the service CA operator. + caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt + # The client's certificate file used by Prometheus when scraping the metrics. + # This file is located in the Prometheus container. + certFile: /etc/prometheus/secrets/metrics-client-certs/tls.crt + # The client's key file used by Prometheus when scraping the metrics. + # This file is located in the Prometheus container. + keyFile: /etc/prometheus/secrets/metrics-client-certs/tls.key + selector: + # Select all Services in the same namespace that have the `app.kubernetes.io/name: my-app` label. + matchLabels: + app: multicluster-observability-alertmanager-metrics \ No newline at end of file diff --git a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-statefulset.yaml b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-statefulset.yaml index ac6ad40b6d..7569a6da67 100644 --- a/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-statefulset.yaml +++ b/operators/multiclusterobservability/manifests/base/alertmanager/alertmanager-statefulset.yaml @@ -2,7 +2,6 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: alertmanager - namespace: open-cluster-management-observability labels: app: multicluster-observability-alertmanager alertmanager: observability @@ -41,127 +40,177 @@ spec: values: - multicluster-observability-alertmanager containers: - - args: - - --config.file=/etc/alertmanager/config/alertmanager.yaml - - --cluster.listen-address=[$(POD_IP)]:9094 - - --storage.path=/alertmanager - - --data.retention=120h - - --web.listen-address=127.0.0.1:9093 - - --web.route-prefix=/ - env: - - name: POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - image: quay.io/stolostron/prometheus-alertmanager:2.3.0-SNAPSHOT-2021-07-26-18-43-26 - imagePullPolicy: IfNotPresent - name: alertmanager - ports: - - containerPort: 9094 - name: mesh-tcp - protocol: TCP - - containerPort: 9093 - name: web - protocol: TCP - - containerPort: 9094 - name: mesh-udp - protocol: UDP - resources: - requests: - cpu: 4m - memory: 200Mi - volumeMounts: - - mountPath: /etc/alertmanager/config - name: config-volume - - mountPath: /alertmanager - name: alertmanager-db - securityContext: - privileged: false - readOnlyRootFilesystem: true - - args: - - -webhook-url=http://localhost:9093/-/reload - - -volume-dir=/etc/alertmanager/config - - -volume-dir=/etc/tls/private - image: quay.io/openshift/origin-configmap-reloader:4.8.0 - imagePullPolicy: IfNotPresent - name: config-reloader - resources: - requests: - cpu: 4m - memory: 25Mi - volumeMounts: - - mountPath: /etc/alertmanager/config - name: config-volume - readOnly: true - - mountPath: /etc/tls/private - name: tls-secret - readOnly: true - securityContext: - privileged: false - readOnlyRootFilesystem: true - - args: - - --provider=openshift - - --https-address=:9095 - - --http-address= - - --upstream=http://localhost:9093 - - --openshift-sar={"resource":"namespaces","verb":"get"} - - --openshift-delegate-urls={"/":{"resource":"namespaces","verb":"get"}} - - --tls-cert=/etc/tls/private/tls.crt - - --tls-key=/etc/tls/private/tls.key - # - --email-domain=* - # - --scope=user:full - # - --client-id=alertmanager - # - --client-secret=alertmanagersecret - - --openshift-service-account=alertmanager - - --cookie-secret-file=/etc/proxy/secrets/session_secret - - --skip-provider-button=true - - --openshift-ca=/etc/pki/tls/cert.pem - - --openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt - image: quay.io/stolostron/origin-oauth-proxy:4.5 - imagePullPolicy: IfNotPresent + - args: + - --config.file=/etc/alertmanager/config/alertmanager.yaml + - --cluster.listen-address=[$(POD_IP)]:9094 + - --storage.path=/alertmanager + - --data.retention=120h + - --web.listen-address=127.0.0.1:9093 + - --web.route-prefix=/ + env: + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + image: quay.io/stolostron/prometheus-alertmanager:2.3.0-SNAPSHOT-2021-07-26-18-43-26 + imagePullPolicy: IfNotPresent + name: alertmanager + ports: + - containerPort: 9094 + name: mesh-tcp + protocol: TCP + - containerPort: 9094 + name: mesh-udp + protocol: UDP + resources: + requests: + cpu: 4m + memory: 200Mi + startupProbe: + exec: + command: + - sh + - -c + - exec curl --fail http://localhost:9093/-/ready + failureThreshold: 40 + initialDelaySeconds: 20 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 3 + volumeMounts: + - mountPath: /etc/alertmanager/config + name: config-volume + - mountPath: /alertmanager + name: alertmanager-db + securityContext: + privileged: false + readOnlyRootFilesystem: true + - args: + - -webhook-url=http://localhost:9093/-/reload + - -volume-dir=/etc/alertmanager/config + - -volume-dir=/etc/tls/private + image: quay.io/openshift/origin-configmap-reloader:4.8.0 + imagePullPolicy: IfNotPresent + name: config-reloader + resources: + requests: + cpu: 4m + memory: 25Mi + volumeMounts: + - mountPath: /etc/alertmanager/config + name: config-volume + readOnly: true + - mountPath: /etc/tls/private + name: tls-secret + readOnly: true + securityContext: + privileged: false + readOnlyRootFilesystem: true + - args: + - --provider=openshift + - --https-address=:9095 + - --http-address= + - --upstream=http://localhost:9093 + - --openshift-sar={"resource":"namespaces","verb":"get"} + - --openshift-delegate-urls={"/":{"resource":"namespaces","verb":"get"}} + - --tls-cert=/etc/tls/private/tls.crt + - --tls-key=/etc/tls/private/tls.key + # - --email-domain=* + # - --scope=user:full + # - --client-id=alertmanager + # - --client-secret=alertmanagersecret + - --openshift-service-account=alertmanager + - --cookie-secret-file=/etc/proxy/secrets/session_secret + - --skip-provider-button=true + - --openshift-ca=/etc/pki/tls/cert.pem + - --openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + image: quay.io/stolostron/origin-oauth-proxy:4.5 + imagePullPolicy: IfNotPresent + name: alertmanager-proxy + ports: + - containerPort: 9095 + name: oauth-proxy + protocol: TCP + resources: + requests: + cpu: 1m + memory: 20Mi + readinessProbe: + failureThreshold: 3 + httpGet: + path: /oauth/healthz + port: 9095 + scheme: HTTPS + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + volumeMounts: + - mountPath: /etc/tls/private + name: tls-secret + readOnly: true + - mountPath: /etc/proxy/secrets name: alertmanager-proxy - ports: - - containerPort: 9095 - name: oauth-proxy - protocol: TCP - resources: - requests: - cpu: 1m - memory: 20Mi - readinessProbe: - failureThreshold: 3 - httpGet: - path: /oauth/healthz - port: 9095 - scheme: HTTPS - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - volumeMounts: - - mountPath: /etc/tls/private - name: tls-secret - readOnly: true - - mountPath: /etc/proxy/secrets - name: alertmanager-proxy - securityContext: - privileged: false - readOnlyRootFilesystem: true + securityContext: + privileged: false + readOnlyRootFilesystem: true + - image: quay.io/stolostron/kube-rbac-proxy:2.10.0-SNAPSHOT-2024-02-13-14-12-35 + name: kube-rbac-proxy + args: + - --secure-listen-address=0.0.0.0:9096 + - --upstream=http://127.0.0.1:9093 + - --config-file=/etc/kube-rbac-proxy/config.yaml + - --tls-cert-file=/etc/tls/private/tls.crt + - --tls-private-key-file=/etc/tls/private/tls.key + - --client-ca-file=/etc/tls/client/client-ca-file + - --logtostderr=true + - --allow-paths=/metrics + ports: + - containerPort: 9096 + name: metrics + protocol: TCP + resources: + requests: + cpu: 10m + memory: 15Mi + volumeMounts: + - mountPath: /etc/tls/private + name: tls-metrics-secret + readOnly: true + - mountPath: /etc/kube-rbac-proxy + name: kube-rbac-proxy-metric + readOnly: true + - mountPath: /etc/tls/client + name: metrics-client-ca + readOnly: true + securityContext: + privileged: false + readOnlyRootFilesystem: true serviceAccount: alertmanager serviceAccountName: alertmanager volumes: - - name: config-volume - secret: - defaultMode: 420 - secretName: alertmanager-config - - name: alertmanager-proxy - secret: - defaultMode: 420 - secretName: alertmanager-proxy - - name: tls-secret - secret: - defaultMode: 420 - secretName: alertmanager-tls + - name: config-volume + secret: + defaultMode: 420 + secretName: alertmanager-config + - name: alertmanager-proxy + secret: + defaultMode: 420 + secretName: alertmanager-proxy + - name: tls-secret + secret: + defaultMode: 420 + secretName: alertmanager-tls + - name: tls-metrics-secret + secret: + defaultMode: 420 + secretName: alertmanager-tls-metrics + - name: kube-rbac-proxy-metric + secret: + secretName: alertmanager-kube-rbac-proxy-metric + - name: metrics-client-ca + configMap: + name: alertmanager-clientca-metric volumeClaimTemplates: - metadata: name: alertmanager-db diff --git a/operators/multiclusterobservability/manifests/base/alertmanager/kustomization.yaml b/operators/multiclusterobservability/manifests/base/alertmanager/kustomization.yaml index b3a2f4445e..d09227bdd5 100644 --- a/operators/multiclusterobservability/manifests/base/alertmanager/kustomization.yaml +++ b/operators/multiclusterobservability/manifests/base/alertmanager/kustomization.yaml @@ -5,6 +5,10 @@ resources: - alertmanager-statefulset.yaml - alertmanager-operated.yaml - alertmanager-service.yaml +- alertmanager-service-metrics.yaml +- alertmanager-servicemonitor.yaml +- alertmanager-clientca-metric.yaml +- alertmanager-kube-rbac-proxy-metric.yaml - alert_rules.yaml - alertmanager-cabundle.yaml - alertmanager-clusterrole.yaml diff --git a/operators/multiclusterobservability/pkg/config/config.go b/operators/multiclusterobservability/pkg/config/config.go index a36b2fc208..127d601d41 100644 --- a/operators/multiclusterobservability/pkg/config/config.go +++ b/operators/multiclusterobservability/pkg/config/config.go @@ -147,6 +147,8 @@ const ( ConfigmapReloaderImgName = "origin-configmap-reloader" ConfigmapReloaderImgTagSuffix = "4.8.0" ConfigmapReloaderKey = "configmap_reloader" + KubeRBACProxyKey = "kube_rbac_proxy" + KubeRBACProxyImgName = "kube-rbac-proxy" OauthProxyImgRepo = "quay.io/stolostron" OauthProxyImgName = "origin-oauth-proxy" @@ -451,11 +453,11 @@ func ReplaceImage(annotations map[string]string, imageRepo, componentName string repoSlice := strings.Split(imageRepo, "/") imageName := strings.Split(repoSlice[len(repoSlice)-1], ":")[0] image := annotationImageRepo + "/" + imageName + ":" + tagSuffix - log.V(1).Info("image replacement", "componentName", image) + log.V(1).Info("image replacement: has tag suffix", "componentName", componentName, "imageRepo", imageRepo, "image", image) return true, image } else if !hasTagSuffix { image, found := imageManifests[componentName] - log.V(1).Info("image replacement", "componentName", image) + log.V(1).Info("image replacement", "componentName", componentName, "image", image) if found { return true, image } @@ -464,7 +466,7 @@ func ReplaceImage(annotations map[string]string, imageRepo, componentName string return false, "" } else { image, found := imageManifests[componentName] - log.V(1).Info("image replacement", "componentName", image) + log.V(1).Info("image replacement", "componentName", componentName, "image", image) if found { return true, image } diff --git a/operators/multiclusterobservability/pkg/rendering/renderer.go b/operators/multiclusterobservability/pkg/rendering/renderer.go index 2343532af9..6a19632d3e 100644 --- a/operators/multiclusterobservability/pkg/rendering/renderer.go +++ b/operators/multiclusterobservability/pkg/rendering/renderer.go @@ -18,11 +18,13 @@ import ( rendererutil "github.com/stolostron/multicluster-observability-operator/operators/pkg/rendering" templatesutil "github.com/stolostron/multicluster-observability-operator/operators/pkg/rendering/templates" "github.com/stolostron/multicluster-observability-operator/operators/pkg/util" + "sigs.k8s.io/controller-runtime/pkg/client" ) var log = logf.Log.WithName("renderer") type MCORenderer struct { + kubeClient client.Client renderer *rendererutil.Renderer cr *obv1beta2.MultiClusterObservability renderGrafanaFns map[string]rendererutil.RenderFn @@ -31,10 +33,11 @@ type MCORenderer struct { renderProxyFns map[string]rendererutil.RenderFn } -func NewMCORenderer(multipleClusterMonitoring *obv1beta2.MultiClusterObservability) *MCORenderer { +func NewMCORenderer(multipleClusterMonitoring *obv1beta2.MultiClusterObservability, kubeClient client.Client) *MCORenderer { mcoRenderer := &MCORenderer{ - renderer: rendererutil.NewRenderer(), - cr: multipleClusterMonitoring, + renderer: rendererutil.NewRenderer(), + cr: multipleClusterMonitoring, + kubeClient: kubeClient, } mcoRenderer.newGranfanaRenderer() mcoRenderer.newAlertManagerRenderer() diff --git a/operators/multiclusterobservability/pkg/rendering/renderer_alertmanager.go b/operators/multiclusterobservability/pkg/rendering/renderer_alertmanager.go index 66747b5cfb..131e515254 100644 --- a/operators/multiclusterobservability/pkg/rendering/renderer_alertmanager.go +++ b/operators/multiclusterobservability/pkg/rendering/renderer_alertmanager.go @@ -5,6 +5,8 @@ package rendering import ( + "context" + "fmt" "strconv" v1 "k8s.io/api/apps/v1" @@ -12,6 +14,7 @@ import ( apiresource "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/kustomize/api/resource" mcoconfig "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/pkg/config" @@ -24,7 +27,7 @@ func (r *MCORenderer) newAlertManagerRenderer() { "StatefulSet": r.renderAlertManagerStatefulSet, "Service": r.renderer.RenderNamespace, "ServiceAccount": r.renderer.RenderNamespace, - "ConfigMap": r.renderer.RenderNamespace, + "ConfigMap": r.renderAlertManagerConfigMap, "ClusterRole": r.renderer.RenderClusterRole, "ClusterRoleBinding": r.renderer.RenderClusterRoleBinding, "Secret": r.renderAlertManagerSecret, @@ -32,6 +35,8 @@ func (r *MCORenderer) newAlertManagerRenderer() { "RoleBinding": r.renderer.RenderNamespace, "Ingress": r.renderer.RenderNamespace, "PersistentVolumeClaim": r.renderer.RenderNamespace, + "ServiceMonitor": r.renderer.RenderNamespace, + "PrometheusRule": r.renderer.RenderNamespace, } } @@ -103,6 +108,16 @@ func (r *MCORenderer) renderAlertManagerStatefulSet(res *resource.Resource, spec.Containers[2].Image = image } spec.Containers[2].ImagePullPolicy = imagePullPolicy + + // fail if kube-rbac-proxy container is not at the expected index + if spec.Containers[3].Name != "kube-rbac-proxy" { + return nil, fmt.Errorf("kube-rbac-proxy container not found in statefulset") + } + if ok, image := mcoconfig.ReplaceImage(r.cr.Annotations, mcoconfig.DefaultImgRepository+"/"+mcoconfig.KubeRBACProxyImgName, mcoconfig.KubeRBACProxyKey); ok { + spec.Containers[3].Image = image + } + spec.Containers[3].ImagePullPolicy = imagePullPolicy + //replace the volumeClaimTemplate dep.Spec.VolumeClaimTemplates[0].Spec.StorageClassName = &r.cr.Spec.StorageConfig.StorageClass dep.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests[corev1.ResourceStorage] = @@ -145,6 +160,55 @@ func (r *MCORenderer) renderAlertManagerSecret(res *resource.Resource, return u, nil } +func (r *MCORenderer) renderAlertManagerConfigMap(res *resource.Resource, + namespace string, labels map[string]string) (*unstructured.Unstructured, error) { + u, err := r.renderer.RenderNamespace(res, namespace, labels) + if err != nil { + return nil, err + } + + if u.GetName() == "alertmanager-clientca-metric" { + cm := &corev1.ConfigMap{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, cm) + if err != nil { + return nil, fmt.Errorf("failed to convert %q to ConfigMap: %w", u.GetName(), err) + } + + // Retrieve the extension-apiserver-authentication ConfigMap from kube-system namespace + namespacedName := types.NamespacedName{ + Name: "extension-apiserver-authentication", + Namespace: "kube-system", + } + sourceConfigMap := &corev1.ConfigMap{} + err = r.kubeClient.Get(context.Background(), namespacedName, sourceConfigMap) + if err != nil { + return nil, fmt.Errorf("error fetching source ConfigMap: %w", err) + } + + // Extract the CA certificate data + caData, exists := sourceConfigMap.Data["client-ca-file"] + if !exists { + return nil, fmt.Errorf("client-ca-file not found in source ConfigMap") + } + + if len(caData) == 0 { + return nil, fmt.Errorf("client-ca-file is empty in source ConfigMap") + } + + // Update the ConfigMap with the CA certificate data + cm.Data["client-ca-file"] = caData + + unstructuredObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(cm) + if err != nil { + return nil, err + } + + return &unstructured.Unstructured{Object: unstructuredObj}, nil + } + + return u, nil +} + func (r *MCORenderer) renderAlertManagerTemplates(templates []*resource.Resource, namespace string, labels map[string]string) ([]*unstructured.Unstructured, error) { uobjs := []*unstructured.Unstructured{} diff --git a/operators/multiclusterobservability/pkg/rendering/renderer_alertmanager_test.go b/operators/multiclusterobservability/pkg/rendering/renderer_alertmanager_test.go new file mode 100644 index 0000000000..9c6cda766a --- /dev/null +++ b/operators/multiclusterobservability/pkg/rendering/renderer_alertmanager_test.go @@ -0,0 +1,288 @@ +package rendering + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + mcoshared "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/shared" + mcov1beta2 "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/v1beta2" + "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/pkg/config" + "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/pkg/rendering/templates" + templatesutil "github.com/stolostron/multicluster-observability-operator/operators/pkg/rendering/templates" + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestAlertManagerRenderer(t *testing.T) { + clientCa := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "extension-apiserver-authentication", + Namespace: "kube-system", + }, + Data: map[string]string{ + "client-ca-file": "test", + }, + } + + containerNameToMchKey := map[string]string{ + "alertmanager": "prometheus_alertmanager", + "config-reloader": "configmap_reloader", + "alertmanager-proxy": "oauth_proxy", + "kube-rbac-proxy": "kube_rbac_proxy", + } + mchImageManifest := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mch-image-manifest", + Namespace: config.GetMCONamespace(), + Labels: map[string]string{ + config.OCMManifestConfigMapTypeLabelKey: config.OCMManifestConfigMapTypeLabelValue, + config.OCMManifestConfigMapVersionLabelKey: "v1", + }, + }, + Data: map[string]string{ + "prometheus_alertmanager": "quay.io/rhacm2/alertmanager:latest", + "configmap_reloader": "quay.io/rhacm2/configmap-reloader:latest", + "oauth_proxy": "quay.io/rhacm2/oauth_proxy:latest", + "kube_rbac_proxy": "quay.io/rhacm2/kube-rbac-proxy:latest", + }, + } + + kubeClient := fake.NewClientBuilder().WithObjects(clientCa, mchImageManifest).Build() + alertResources := renderTemplates(t, kubeClient, makeBaseMco()) + + // clientCa configmap must be filled with the client-ca-file data + clientCaData := getResource[*corev1.ConfigMap](alertResources, "alertmanager-clientca-metric") + assert.Equal(t, clientCa.Data["client-ca-file"], clientCaData.Data["client-ca-file"]) + + // container images must be replaced with the ones from the mch-image-manifest configmap + for _, obj := range alertResources { + if obj.GetKind() == "StatefulSet" { // there is only one statefulset + sts := &appsv1.StatefulSet{} + runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, sts) + for _, container := range sts.Spec.Template.Spec.Containers { + assert.Equal(t, mchImageManifest.Data[containerNameToMchKey[container.Name]], container.Image) + } + } + } + + // namespace must be set to the one provided in the arguments + for _, obj := range alertResources { + if obj.GetKind() == "ClusterRole" || obj.GetKind() == "ClusterRoleBinding" { + continue + } + + if obj.GetName() == "acm-observability-alert-rules" { // has no update annotation + continue + } + + assert.Equal(t, "namespace", obj.GetNamespace(), fmt.Sprintf("kind: %s, name: %s", obj.GetKind(), obj.GetName())) + } + + // alertmanager-proxy must have the secret value generated + proxy := getResource[*corev1.Secret](alertResources, "alertmanager-proxy") + assert.True(t, len(proxy.Data["session_secret"]) > 0) + +} + +func TestAlertManagerRendererMCOConfig(t *testing.T) { + testCases := map[string]struct { + mco func() *mcov1beta2.MultiClusterObservability + expect func(*testing.T, *appsv1.StatefulSet) + }{ + "storage": { + mco: func() *mcov1beta2.MultiClusterObservability { + ret := makeBaseMco() + ret.Spec.StorageConfig.AlertmanagerStorageSize = "12Gi" + ret.Spec.StorageConfig.StorageClass = "mystorage" + return ret + }, + expect: func(t *testing.T, sts *appsv1.StatefulSet) { + qty := sts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests[corev1.ResourceStorage] + assert.Equal(t, "12Gi", qty.String()) + assert.Equal(t, "mystorage", *sts.Spec.VolumeClaimTemplates[0].Spec.StorageClassName) + }, + }, + "imagePullPolicy": { + mco: func() *mcov1beta2.MultiClusterObservability { + ret := makeBaseMco() + ret.Spec.ImagePullPolicy = corev1.PullNever + return ret + }, + expect: func(t *testing.T, sts *appsv1.StatefulSet) { + for _, container := range sts.Spec.Template.Spec.Containers { + assert.Equal(t, corev1.PullNever, container.ImagePullPolicy) + } + }, + }, + "imagePullSecret": { + mco: func() *mcov1beta2.MultiClusterObservability { + ret := makeBaseMco() + ret.Spec.ImagePullSecret = "mysecret" + return ret + }, + expect: func(t *testing.T, sts *appsv1.StatefulSet) { + assert.Equal(t, "mysecret", sts.Spec.Template.Spec.ImagePullSecrets[0].Name) + }, + }, + "more than one replicas": { + mco: func() *mcov1beta2.MultiClusterObservability { + ret := makeBaseMco() + replicas := int32(3) + ret.Spec.AdvancedConfig = &mcov1beta2.AdvancedConfig{ + Alertmanager: &mcov1beta2.CommonSpec{ + Replicas: &replicas, + }, + } + return ret + }, + expect: func(t *testing.T, sts *appsv1.StatefulSet) { + assert.Equal(t, int32(3), *sts.Spec.Replicas) + // args must contain --cluster.peer flag with the correct number of replicas + args := sts.Spec.Template.Spec.Containers[0].Args + count := 0 + for _, arg := range args { + if strings.Contains(arg, "--cluster.peer") { + count++ + } + } + assert.Equal(t, 3, count) + }, + }, + "resources": { + mco: func() *mcov1beta2.MultiClusterObservability { + ret := makeBaseMco() + ret.Spec.AdvancedConfig = &mcov1beta2.AdvancedConfig{ + Alertmanager: &mcov1beta2.CommonSpec{ + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("500Mi"), + }, + }, + }, + } + return ret + }, + expect: func(t *testing.T, sts *appsv1.StatefulSet) { + assert.Equal(t, resource.MustParse("1"), sts.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceCPU]) + assert.Equal(t, resource.MustParse("1Gi"), sts.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceMemory]) + assert.Equal(t, resource.MustParse("500m"), sts.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU]) + assert.Equal(t, resource.MustParse("500Mi"), sts.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory]) + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + clientCa := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "extension-apiserver-authentication", + Namespace: "kube-system", + }, + Data: map[string]string{ + "client-ca-file": "test", + }, + } + kubeClient := fake.NewClientBuilder().WithObjects(clientCa).Build() + + alertResources := renderTemplates(t, kubeClient, tc.mco()) + + sts := getResource[*appsv1.StatefulSet](alertResources, "") + tc.expect(t, sts) + }) + } +} + +func makeBaseMco() *mcov1beta2.MultiClusterObservability { + return &mcov1beta2.MultiClusterObservability{ + TypeMeta: metav1.TypeMeta{Kind: "MultiClusterObservability"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "test"}, + Spec: mcov1beta2.MultiClusterObservabilitySpec{ + StorageConfig: &mcov1beta2.StorageConfig{ + MetricObjectStorage: &mcoshared.PreConfiguredStorage{ + Key: "test", + Name: "test", + }, + StorageClass: "gp2", + AlertmanagerStorageSize: "1Gi", + }, + }, + } +} + +func renderTemplates(t *testing.T, kubeClient client.Client, mco *mcov1beta2.MultiClusterObservability) []*unstructured.Unstructured { + wd, err := os.Getwd() + assert.NoError(t, err) + templatesPath := filepath.Join(wd, "..", "..", "manifests") + os.Setenv(templatesutil.TemplatesPathEnvVar, templatesPath) + defer os.Unsetenv(templatesutil.TemplatesPathEnvVar) + + config.ReadImageManifestConfigMap(kubeClient, "v1") + renderer := NewMCORenderer(mco, kubeClient) + + //load and render alertmanager templates + alertTemplates, err := templates.GetOrLoadAlertManagerTemplates(templatesutil.GetTemplateRenderer()) + assert.NoError(t, err) + alertResources, err := renderer.renderAlertManagerTemplates(alertTemplates, "namespace", map[string]string{"test": "test"}) + assert.NoError(t, err) + + return alertResources +} + +func getResource[T runtime.Object](objects []*unstructured.Unstructured, name string) T { + var ret T + found := false + + if reflect.TypeOf(ret).Kind() != reflect.Ptr { // Ensure it's a pointer + panic(fmt.Sprintf("expected a pointer, got %T", ret)) + } + tType := reflect.TypeOf(ret).Elem() // Get the type of the object + + for _, obj := range objects { + // check if the name matches + if name != "" && obj.GetName() != name { + continue + } + + // convert unstructured object to typed object + typedObject := reflect.New(tType).Interface().(T) + err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, typedObject) + if err != nil { + panic(fmt.Sprintf("failed to convert %q to %T: %v", obj.GetName(), *new(T), err)) + } + + // check if the kind matches + if !strings.Contains(tType.String(), obj.GetKind()) { + continue + } + + // check if we already found an object of this type. If so, panic + if found { + panic(fmt.Sprintf("found multiple objects of type %T", *new(T))) + } + + found = true + ret = typedObject + } + + if !found { + panic(fmt.Sprintf("could not find object of type %T", *new(T))) + } + + return ret +} diff --git a/operators/multiclusterobservability/pkg/rendering/renderer_test.go b/operators/multiclusterobservability/pkg/rendering/renderer_test.go index 57cfe57969..49d26c3053 100644 --- a/operators/multiclusterobservability/pkg/rendering/renderer_test.go +++ b/operators/multiclusterobservability/pkg/rendering/renderer_test.go @@ -9,12 +9,14 @@ import ( "path" "testing" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" mcoshared "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/shared" mcov1beta2 "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/v1beta2" templatesutil "github.com/stolostron/multicluster-observability-operator/operators/pkg/rendering/templates" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestRender(t *testing.T) { @@ -47,7 +49,18 @@ func TestRender(t *testing.T) { }, } - renderer := NewMCORenderer(mchcr) + clientCa := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "extension-apiserver-authentication", + Namespace: "kube-system", + }, + Data: map[string]string{ + "client-ca-file": "test", + }, + } + kubeClient := fake.NewClientBuilder().WithObjects(clientCa).Build() + + renderer := NewMCORenderer(mchcr, kubeClient) objs, err := renderer.Render() if err != nil { t.Fatalf("failed to render MultiClusterObservability: %v", err) diff --git a/scripts/install-binaries.sh b/scripts/install-binaries.sh index 7bbcb76782..96ce3b668d 100755 --- a/scripts/install-binaries.sh +++ b/scripts/install-binaries.sh @@ -12,6 +12,7 @@ OPERATOR_SDK_VERSION="${KUBECTL_VERSION:=v1.4.2}" KUBECTL_VERSION="${KUBECTL_VERSION:=v1.28.2}" KUSTOMIZE_VERSION="${KUSTOMIZE_VERSION:=v5.3.0}" JQ_VERSION="${JQ_VERSION:=1.6}" +KIND_VERSION="${KIND_VERSION:=v0.22.0}" BIN_DIR="${BIN_DIR:=/usr/local/bin}" @@ -59,6 +60,7 @@ install_kustomize() { install_jq() { bin_dir=${1:-${BIN_DIR}} if ! command -v jq &>/dev/null; then + echo "This script will install jq on your machine" if [[ "$(uname)" == "Linux" ]]; then curl -o jq -L "https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64" elif [[ "$(uname)" == "Darwin" ]]; then @@ -68,6 +70,19 @@ install_jq() { fi } +install_kind() { + bin_dir=${1:-${BIN_DIR}} + if ! command -v kind &>/dev/null; then + echo "This script will install KinD on your machine" + if [[ "$(uname)" == "Linux" ]]; then + curl -o kind -L "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" + elif [[ "$(uname)" == "Darwin" ]]; then + curl -o kind -L "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-darwin-$(uname -m)" + fi + chmod +x ./kind && mv ./kind ${bin_dir}/kind + fi +} + install_build_deps() { bin_dir=${1:-${BIN_DIR}} install_operator_sdk ${bin_dir} @@ -75,5 +90,13 @@ install_build_deps() { install_kustomize ${bin_dir} } +install_e2e_tests_deps() { + bin_dir=${1:-${BIN_DIR}} + install_kubectl ${bin_dir} + install_jq ${bin_dir} + install_kind ${bin_dir} + install_kustomize ${bin_dir} +} + # This allows functions within this file to be called individually from Makefile(s). $* diff --git a/scripts/test-utils.sh b/scripts/test-utils.sh new file mode 100755 index 0000000000..d4a3654ce9 --- /dev/null +++ b/scripts/test-utils.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright (c) 2024 Red Hat, Inc. +# Copyright Contributors to the Open Cluster Management project + +# Use snapshot for target release. +# Use latest if no branch info detected, or not a release branch. +get_latest_snapshot() { + BRANCH="" + LATEST_SNAPSHOT="" + if [[ ${PULL_BASE_REF} == "release-"* ]]; then + BRANCH=${PULL_BASE_REF#"release-"} + LATEST_SNAPSHOT=$(curl https://quay.io//api/v1/repository/open-cluster-management/multicluster-observability-operator | jq '.tags|with_entries(select(.key|test("'${BRANCH}'.*-SNAPSHOT-*")))|keys[length-1]') + fi + if [[ ${LATEST_SNAPSHOT} == "null" ]] || [[ ${LATEST_SNAPSHOT} == "" ]]; then + LATEST_SNAPSHOT=$(curl https://quay.io/api/v1/repository/stolostron/multicluster-observability-operator | jq '.tags|with_entries(select((.key|contains("SNAPSHOT"))and(.key|contains("9.9.0")|not)))|keys[length-1]') + fi + + # trim the leading and tailing quotes + LATEST_SNAPSHOT="${LATEST_SNAPSHOT#\"}" + LATEST_SNAPSHOT="${LATEST_SNAPSHOT%\"}" + echo ${LATEST_SNAPSHOT} +} diff --git a/tests/run-in-kind/req_crds/servicemonitor-crd.json b/tests/run-in-kind/req_crds/servicemonitor-crd.json index e47cb2f2c2..f8683e129b 100644 --- a/tests/run-in-kind/req_crds/servicemonitor-crd.json +++ b/tests/run-in-kind/req_crds/servicemonitor-crd.json @@ -777,4 +777,4 @@ } ] } - } \ No newline at end of file + } diff --git a/tests/run-in-kind/run-e2e-in-kind.sh b/tests/run-in-kind/run-e2e-in-kind.sh index 66e58f10e3..49d36ec295 100755 --- a/tests/run-in-kind/run-e2e-in-kind.sh +++ b/tests/run-in-kind/run-e2e-in-kind.sh @@ -13,26 +13,7 @@ export IS_KIND_ENV=true # shellcheck disable=SC1091 source ${WORKDIR}/env.sh -setup_kubectl_command() { - if ! command -v kubectl >/dev/null 2>&1; then - echo "This script will install kubectl (https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your machine" - if [[ "$(uname)" == "Linux" ]]; then - curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl - elif [[ "$(uname)" == "Darwin" ]]; then - curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/darwin/amd64/kubectl - fi - chmod +x ./kubectl - sudo mv ./kubectl /usr/local/bin/kubectl - fi -} - create_kind_cluster() { - if ! command -v kind >/dev/null 2>&1; then - echo "This script will install kind (https://kind.sigs.k8s.io/) on your machine." - curl -Lo ./kind-amd64 "https://kind.sigs.k8s.io/dl/v0.10.0/kind-$(uname)-amd64" - chmod +x ./kind-amd64 - sudo mv ./kind-amd64 /usr/local/bin/kind - fi echo "Delete the KinD cluster if exists" kind delete cluster --name $1 || true rm -rf $HOME/.kube/kind-config-$1 @@ -69,7 +50,6 @@ run_e2e_test() { } run() { - setup_kubectl_command create_kind_cluster hub deploy_crds deploy_templates diff --git a/tools/simulator/README.md b/tools/simulator/README.md index e74fbe2c37..4adbda77be 100644 --- a/tools/simulator/README.md +++ b/tools/simulator/README.md @@ -1,6 +1,7 @@ # Simulator -We have simulators in different subfolder for different scenarios, each simulator subfolder contains its own README, They're linked below! +We have simulators in different subfolder for different scenarios. +Each simulator subfolder contains its own README linked below: * [alert forwarder simulator](alert-forward) * [managed cluster simulator](managed-cluster) diff --git a/tools/simulator/managed-cluster/README.md b/tools/simulator/managed-cluster/README.md index a104135f48..b390c40682 100644 --- a/tools/simulator/managed-cluster/README.md +++ b/tools/simulator/managed-cluster/README.md @@ -1,70 +1,66 @@ # Managed Cluster Simulator -The managed cluster simulator can be used to set up multiple managed clusters and create the corresponding namespaces in ACM hub cluster, to simulate reconciling thousands of managed clusters for the multicluster-observability-operator. +The managed cluster simulator can be used to set up multiple managed clusters and create the corresponding namespaces +in ACM hub cluster, to simulate reconciling thousands of managed clusters for the multicluster-observability-operator. _Note:_ this simulator is for testing purpose only. -## Prereqs +## Prerequisites -You must meet the following requirements to setup managed cluster simulator: +The following are requirements to set up the managed cluster simulator: -1. ACM 2.3+ available -2. `MultiClusterObservability` instance available in the hub cluster +1. `ACM` version 2.3+ +2. `MultiClusterObservability` instance available in the Hub cluster +3. `kubectl` ## How to use ### Set up managed cluster simulator -1. You can run `setup-managedcluster.sh` followed with two numbers(start index and end index) to set up multiple simulated managed clusters. For example, set up 1-5 simulated managedcluster with the following command: +You can run `setup-managedcluster.sh` followed by two numbers (start and end index) to set up +multiple simulated managed clusters. For example, set up five simulated `managedcluster` with the following command: ```bash -# ./setup-managedcluster.sh 1 5 -Creating Simulated managedCluster simulated-1-managedcluster... -managedcluster.cluster.open-cluster-management.io/simulated-1-managedcluster created -Creating Simulated managedCluster simulated-2-managedcluster... -managedcluster.cluster.open-cluster-management.io/simulated-2-managedcluster created -Creating Simulated managedCluster simulated-3-managedcluster... -managedcluster.cluster.open-cluster-management.io/simulated-3-managedcluster created -Creating Simulated managedCluster simulated-4-managedcluster... -managedcluster.cluster.open-cluster-management.io/simulated-4-managedcluster created -Creating Simulated managedCluster simulated-5-managedcluster... -managedcluster.cluster.open-cluster-management.io/simulated-5-managedcluster created +./setup-managedcluster.sh 1 5 ``` -2. Check if all the managed cluster are set up successfully in ACM hub cluster: +Check if all the `managedcluster` are set up successfully in ACM hub cluster: ```bash -$ oc get managedcluster | grep simulated -simulated-1-managedcluster true 46s -simulated-2-managedcluster true 46s -simulated-3-managedcluster true 45s -simulated-4-managedcluster true 44s -simulated-5-managedcluster true 44s +$ kubectl get managedcluster | grep simulated +# simulated-1-managedcluster true 46s +# simulated-2-managedcluster true 46s +# simulated-3-managedcluster true 45s +# simulated-4-managedcluster true 44s +# simulated-5-managedcluster true 44s ``` -3. Check if the `Manifestwork` are created for the simulated managed clusters: +Check if the `Manifestwork` are created for the simulated managed clusters: ```bash -$ for i in $(seq 1 5); do oc -n simulated-$i-managedcluster get manifestwork --no-headers; done -simulated-1-managedcluster-observability 72s -simulated-2-managedcluster-observability 70s -simulated-3-managedcluster-observability 69s -simulated-4-managedcluster-observability 67s -simulated-5-managedcluster-observability 65s +$ for i in $(seq 1 5); do kubectl -n simulated-$i-managedcluster get manifestwork --no-headers; done +# simulated-1-managedcluster-observability 72s +# simulated-2-managedcluster-observability 70s +# simulated-3-managedcluster-observability 69s +# simulated-4-managedcluster-observability 67s +# simulated-5-managedcluster-observability 65s ``` -4. Clean up the simulated managed clusters by running the `clean-managedcluster.sh` script followed with two numbers(start index and end index), For example, clean up 1-5 simulated managedcluster with the following command: +Clean up the simulated `managedclusters` by running the +`clean-managedcluster.sh` script followed by two numbers (start and end index). -``` +For example, clean up the previously created five simulated `managedclusters` with the following command: + +```bash $ ./clean-managedcluster.sh 1 5 -Deleting Simulated managedCluster simulated-1-managedcluster... -managedcluster.cluster.open-cluster-management.io "simulated-1-managedcluster" deleted -Deleting Simulated managedCluster simulated-2-managedcluster... -managedcluster.cluster.open-cluster-management.io "simulated-2-managedcluster" deleted -Deleting Simulated managedCluster simulated-3-managedcluster... -managedcluster.cluster.open-cluster-management.io "simulated-3-managedcluster" deleted -Deleting Simulated managedCluster simulated-4-managedcluster... -managedcluster.cluster.open-cluster-management.io "simulated-4-managedcluster" deleted -Deleting Simulated managedCluster simulated-5-managedcluster... -managedcluster.cluster.open-cluster-management.io "simulated-5-managedcluster" deleted +# Deleting Simulated managedCluster simulated-1-managedcluster... +# managedcluster.cluster.open-cluster-management.io "simulated-1-managedcluster" deleted +# Deleting Simulated managedCluster simulated-2-managedcluster... +# managedcluster.cluster.open-cluster-management.io "simulated-2-managedcluster" deleted +# Deleting Simulated managedCluster simulated-3-managedcluster... +# managedcluster.cluster.open-cluster-management.io "simulated-3-managedcluster" deleted +# Deleting Simulated managedCluster simulated-4-managedcluster... +# managedcluster.cluster.open-cluster-management.io "simulated-4-managedcluster" deleted +# Deleting Simulated managedCluster simulated-5-managedcluster... +# managedcluster.cluster.open-cluster-management.io "simulated-5-managedcluster" deleted ``` diff --git a/tools/simulator/managed-cluster/setup-managedcluster.sh b/tools/simulator/managed-cluster/setup-managedcluster.sh index 35e282e734..b90bf3c2ed 100755 --- a/tools/simulator/managed-cluster/setup-managedcluster.sh +++ b/tools/simulator/managed-cluster/setup-managedcluster.sh @@ -4,33 +4,10 @@ set -exo pipefail -WORK_DIR="$( - cd "$(dirname "$0")" - pwd -P -)" -# Create bin directory and add it to PATH -mkdir -p ${WORK_DIR}/bin -export PATH=${PATH}:${WORK_DIR}/bin - -KUBECTL="kubectl" -if ! command -v kubectl &>/dev/null; then - if command -v oc &>/dev/null; then - KUBECTL="oc" - else - echo "This script will install kubectl (https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your machine" - if [[ "$(uname)" == "Linux" ]]; then - curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl - elif [[ "$(uname)" == "Darwin" ]]; then - curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/darwin/amd64/kubectl - fi - chmod +x ./kubectl && mv ./kubectl ${WORK_DIR}/bin/kubectl - fi -fi - # creating the simulated managedcluster for i in $(seq $1 $2); do echo "Creating Simulated managedCluster simulated-${i}-managedcluster..." - cat <