Horreum: Include latencies from Infinispan Metrics #636 (#749)

Signed-off-by: Anna Manukyan <amanukya@redhat.com>

.github/actions/prometheus-metrics-calc/action.yml

@@ -99,10 +99,17 @@ runs:
         gatlingReportDirName=$(ls -lrt $gatlingReportPath | tail -1 | awk '{print $9}')
         gatlingReportFilePath="$gatlingReportPath$gatlingReportDirName/js/stats.json"
         statsJson=$(jq '.. | .stats? | select( . != null )' $gatlingReportFilePath | jq -s .)
-        #Storing all information in JSON using jq library
-        jq --arg testName "${{ inputs.performedTestName }}" '.[$testName] += {"statistics":[]}' ${HORREUM_OUTPUT_FILE_NAME} > tmp.json && \
-        mv tmp.json ${HORREUM_OUTPUT_FILE_NAME}
         #Putting environment parameters into JSON
         jq --argjson statsJson "${statsJson}" --arg testName "${{ inputs.performedTestName }}" \
-        '.[$testName].statistics = ($statsJson)' ${HORREUM_OUTPUT_FILE_NAME} > tmp.json && \
+        '.[$testName].statistics |= .+ ($statsJson)' ${HORREUM_OUTPUT_FILE_NAME} > tmp.json && \
         mv tmp.json ${HORREUM_OUTPUT_FILE_NAME}
+        #Reading ispn metrics file
+        readarray -t lines < ispn_metrics_file
+        for i in "${lines[@]}"
+        do
+            IFS=',' read -ra metric <<< "$i"
+            #Putting metric into JSON
+            jq --arg cacheName "${metric[0]}" --arg val "${metric[1]}" --arg testName "${{ inputs.performedTestName }}" \
+            '.[$testName].ispnStatistics |= .+ {"cacheName": ($cacheName), "averageXsiteReplicationTime":($val|tonumber)}' ${HORREUM_OUTPUT_FILE_NAME} > tmp.json && \
+            mv tmp.json ${HORREUM_OUTPUT_FILE_NAME}
+        done

.github/actions/prometheus-run-queries/action.yml

@@ -11,6 +11,9 @@ inputs:
   runMemoryUsageTotal:
     description: 'Identifies if "Memory Usage Total" query should be run.'
     default: 'false'
+  ispnMetrics:
+    description: 'Identifies if ISPN metrics should be retrieved.'
+    default: 'false'
   output:
     description: 'The name of the output to store data in'
     default: 'out'
@@ -53,3 +56,18 @@ runs:
       env:
         THANOS_HOST: ${{ env.THANOS_HOST }}
         OC_TOKEN: ${{ env.OC_TOKEN }}
+
+    - id: ispn-metrics
+      name: ISPN response information
+      if: ${{ inputs.ispnMetrics == 'true' }}
+      shell: bash
+      # language=bash
+      run: |
+        cacheName="sessions"
+        ispnMetric=$(curl -s -H "Authorization: Bearer $OC_TOKEN" -k "https://$THANOS_HOST/api/v1/query" --data-urlencode \
+        "query=avg(vendor_rpc_manager_average_xsite_replication_time{job='infinispan-admin', namespace='${{ inputs.project }}',container='infinispan', cache='$cacheName'})" \
+        | jq '.data.result[0].value[1]' -r)
+        echo "${cacheName},${ispnMetric}" >> ispn_metrics_file
+      env:
+        THANOS_HOST: ${{ env.THANOS_HOST }}
+        OC_TOKEN: ${{ env.OC_TOKEN }}

.github/workflows/rosa-multi-az-cluster-undeploy.yml

@@ -53,13 +53,18 @@ jobs:
         with:
           project: runner-keycloak
 
+      - name: Get KC Health URL
+        run: |
+          echo "KC_HEALTH_URL_CLUSTER_1=primary.${KEYCLOAK_URL:15}" >> $GITHUB_ENV
+
       - name: Undeploy infinispan and keycloak
         working-directory: provision/rosa-cross-dc
         run: |
           task undeploy
         env:
           ROSA_CLUSTER_NAME_1: ${{ env.CLUSTER_PREFIX }}-a
           ROSA_CLUSTER_NAME_2: ${{ env.CLUSTER_PREFIX }}-b
+          KC_HEALTH_URL_CLUSTER_1: ${{ env.KC_HEALTH_URL_CLUSTER_1 }}
 
       - name: Undeploy aurora database
         if: ${{ !inputs.skipAuroraDeletion }}

.github/workflows/rosa-scaling-benchmark.yml

@@ -233,6 +233,7 @@ jobs:
         with:
           project: ${{ env.PROJECT }}
           runMemoryUsageTotal: true
+          ispnMetrics: true
           output: memory_create_sessions
 
       - name: Calculate number of active sessions
@@ -281,6 +282,7 @@ jobs:
         with:
           project: ${{ env.PROJECT }}
           runCpuSecsUtil: true
+          ispnMetrics: true
           output: user_logins_vCpu
 
       - name: Calculate and Store CPU sec Util For Ran Benchmark
@@ -322,6 +324,7 @@ jobs:
         with:
           project: ${{ env.PROJECT }}
           runCpuSecsUtil: true
+          ispnMetrics: true
           output: client_credential_grants_vCpu
 
       - name: Calculate and Store CPU usage For Ran Benchmark

provision/infinispan/ispn-helm/templates/infinispan.yaml

@@ -47,6 +47,8 @@ metadata:
     infinispan.org/monitoring: 'true' # <2>
 spec:
   replicas: {{ .Values.replicas }}
+  jmx:
+    enabled: true
 # end::infinispan-single[]
 # end::infinispan-crossdc[]
   # This exposes the http endpoint to interact with its caches - more info - https://infinispan.org/docs/stable/titles/rest/rest.html