From 9f04a2fc12b5d10b1274e6987379251a926a7ffd Mon Sep 17 00:00:00 2001 From: andyuk1986 Date: Tue, 2 Apr 2024 11:53:20 +0200 Subject: [PATCH] Horreum: Include latencies from Infinispan Metrics #636 (#749) Signed-off-by: Anna Manukyan --- .../actions/prometheus-metrics-calc/action.yml | 15 +++++++++++---- .../actions/prometheus-run-queries/action.yml | 18 ++++++++++++++++++ .../rosa-multi-az-cluster-undeploy.yml | 5 +++++ .github/workflows/rosa-scaling-benchmark.yml | 3 +++ .../ispn-helm/templates/infinispan.yaml | 2 ++ 5 files changed, 39 insertions(+), 4 deletions(-) diff --git a/.github/actions/prometheus-metrics-calc/action.yml b/.github/actions/prometheus-metrics-calc/action.yml index 9c1cd6f1..3c38b21c 100644 --- a/.github/actions/prometheus-metrics-calc/action.yml +++ b/.github/actions/prometheus-metrics-calc/action.yml @@ -99,10 +99,17 @@ runs: gatlingReportDirName=$(ls -lrt $gatlingReportPath | tail -1 | awk '{print $9}') gatlingReportFilePath="$gatlingReportPath$gatlingReportDirName/js/stats.json" statsJson=$(jq '.. | .stats? | select( . != null )' $gatlingReportFilePath | jq -s .) - #Storing all information in JSON using jq library - jq --arg testName "${{ inputs.performedTestName }}" '.[$testName] += {"statistics":[]}' ${HORREUM_OUTPUT_FILE_NAME} > tmp.json && \ - mv tmp.json ${HORREUM_OUTPUT_FILE_NAME} #Putting environment parameters into JSON jq --argjson statsJson "${statsJson}" --arg testName "${{ inputs.performedTestName }}" \ - '.[$testName].statistics = ($statsJson)' ${HORREUM_OUTPUT_FILE_NAME} > tmp.json && \ + '.[$testName].statistics |= .+ ($statsJson)' ${HORREUM_OUTPUT_FILE_NAME} > tmp.json && \ mv tmp.json ${HORREUM_OUTPUT_FILE_NAME} + #Reading ispn metrics file + readarray -t lines < ispn_metrics_file + for i in "${lines[@]}" + do + IFS=',' read -ra metric <<< "$i" + #Putting metric into JSON + jq --arg cacheName "${metric[0]}" --arg val "${metric[1]}" --arg testName "${{ inputs.performedTestName }}" \ + '.[$testName].ispnStatistics |= .+ {"cacheName": ($cacheName), "averageXsiteReplicationTime":($val|tonumber)}' ${HORREUM_OUTPUT_FILE_NAME} > tmp.json && \ + mv tmp.json ${HORREUM_OUTPUT_FILE_NAME} + done diff --git a/.github/actions/prometheus-run-queries/action.yml b/.github/actions/prometheus-run-queries/action.yml index 8f8844fb..4bf7d7ea 100644 --- a/.github/actions/prometheus-run-queries/action.yml +++ b/.github/actions/prometheus-run-queries/action.yml @@ -11,6 +11,9 @@ inputs: runMemoryUsageTotal: description: 'Identifies if "Memory Usage Total" query should be run.' default: 'false' + ispnMetrics: + description: 'Identifies if ISPN metrics should be retrieved.' + default: 'false' output: description: 'The name of the output to store data in' default: 'out' @@ -53,3 +56,18 @@ runs: env: THANOS_HOST: ${{ env.THANOS_HOST }} OC_TOKEN: ${{ env.OC_TOKEN }} + + - id: ispn-metrics + name: ISPN response information + if: ${{ inputs.ispnMetrics == 'true' }} + shell: bash + # language=bash + run: | + cacheName="sessions" + ispnMetric=$(curl -s -H "Authorization: Bearer $OC_TOKEN" -k "https://$THANOS_HOST/api/v1/query" --data-urlencode \ + "query=avg(vendor_rpc_manager_average_xsite_replication_time{job='infinispan-admin', namespace='${{ inputs.project }}',container='infinispan', cache='$cacheName'})" \ + | jq '.data.result[0].value[1]' -r) + echo "${cacheName},${ispnMetric}" >> ispn_metrics_file + env: + THANOS_HOST: ${{ env.THANOS_HOST }} + OC_TOKEN: ${{ env.OC_TOKEN }} diff --git a/.github/workflows/rosa-multi-az-cluster-undeploy.yml b/.github/workflows/rosa-multi-az-cluster-undeploy.yml index 64b55338..c4b884f2 100644 --- a/.github/workflows/rosa-multi-az-cluster-undeploy.yml +++ b/.github/workflows/rosa-multi-az-cluster-undeploy.yml @@ -53,6 +53,10 @@ jobs: with: project: runner-keycloak + - name: Get KC Health URL + run: | + echo "KC_HEALTH_URL_CLUSTER_1=primary.${KEYCLOAK_URL:15}" >> $GITHUB_ENV + - name: Undeploy infinispan and keycloak working-directory: provision/rosa-cross-dc run: | @@ -60,6 +64,7 @@ jobs: env: ROSA_CLUSTER_NAME_1: ${{ env.CLUSTER_PREFIX }}-a ROSA_CLUSTER_NAME_2: ${{ env.CLUSTER_PREFIX }}-b + KC_HEALTH_URL_CLUSTER_1: ${{ env.KC_HEALTH_URL_CLUSTER_1 }} - name: Undeploy aurora database if: ${{ !inputs.skipAuroraDeletion }} diff --git a/.github/workflows/rosa-scaling-benchmark.yml b/.github/workflows/rosa-scaling-benchmark.yml index 373602f1..908788f9 100644 --- a/.github/workflows/rosa-scaling-benchmark.yml +++ b/.github/workflows/rosa-scaling-benchmark.yml @@ -233,6 +233,7 @@ jobs: with: project: ${{ env.PROJECT }} runMemoryUsageTotal: true + ispnMetrics: true output: memory_create_sessions - name: Calculate number of active sessions @@ -281,6 +282,7 @@ jobs: with: project: ${{ env.PROJECT }} runCpuSecsUtil: true + ispnMetrics: true output: user_logins_vCpu - name: Calculate and Store CPU sec Util For Ran Benchmark @@ -322,6 +324,7 @@ jobs: with: project: ${{ env.PROJECT }} runCpuSecsUtil: true + ispnMetrics: true output: client_credential_grants_vCpu - name: Calculate and Store CPU usage For Ran Benchmark diff --git a/provision/infinispan/ispn-helm/templates/infinispan.yaml b/provision/infinispan/ispn-helm/templates/infinispan.yaml index 28488377..1f65505b 100644 --- a/provision/infinispan/ispn-helm/templates/infinispan.yaml +++ b/provision/infinispan/ispn-helm/templates/infinispan.yaml @@ -47,6 +47,8 @@ metadata: infinispan.org/monitoring: 'true' # <2> spec: replicas: {{ .Values.replicas }} + jmx: + enabled: true # end::infinispan-single[] # end::infinispan-crossdc[] # This exposes the http endpoint to interact with its caches - more info - https://infinispan.org/docs/stable/titles/rest/rest.html