Create JSON to describe setup and results, upload to result branch an…

…d create a dashboard from it Closes #373
keycloak · Jun 23, 2023 · fe58cf8 · fe58cf8
1 parent 26def19
commit fe58cf8
Show file tree

Hide file tree

Showing 15 changed files with 642 additions and 6 deletions.
diff --git a/.github/actions/oc-keycloak-login/action.yml b/.github/actions/oc-keycloak-login/action.yml
@@ -15,3 +15,8 @@ runs:
       working-directory: provision/aws
       env:
         CLUSTER_NAME: ${{ inputs.clusterName }}
+
+    - id: enable-self-describe
+      shell: bash
+      working-directory: provision/aws
+      run: echo "SUT_DESCRIBE=$(pwd)/rosa_describe.sh ${{ inputs.clusterName }}" >> "$GITHUB_ENV"
diff --git a/.github/workflows/docs-pages.yml b/.github/workflows/docs-pages.yml
@@ -1,12 +1,19 @@
 name: Publishing Documentation Site
 
 on:
+  workflow_dispatch:
+  workflow_run:
+    workflows:
+      - "Aggregate Benchmark results"
+    types:
+      - completed
   push:
     branches:
       - main
     paths:
       - '.github/workflows/docs-pages.yml'
       - 'antora/**'
+      - 'dashbuilder/**'
       - 'doc/**'
 
 concurrency:
@@ -19,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Node.js
+      - name: Set up Node.js for Antora
         uses: actions/setup-node@v3
         with:
           node-version: '16.x'
@@ -28,6 +35,23 @@ jobs:
       - name: Build docs
         working-directory: antora
         run: ./build.sh
+      - name: Add benchmark data to website
+        run: |
+          mkdir -p antora/_site/keycloak-benchmark/results
+          gh release download aggregated-benchmarks -D antora/_site/keycloak-benchmark/results --repo keycloak/keycloak-benchmark
+        env:
+          GH_TOKEN: ${{ github.token }}
+      - name: Set up Node.js for DashBuilder
+        uses: actions/setup-node@v3
+        with:
+          node-version: '16.x'
+          cache: 'yarn'
+          cache-dependency-path: 'dashbuilder/yarn.lock'
+      - name: Prepare dashbuilder
+        working-directory: dashbuilder
+        run: |
+          ./build.sh
+          mv dist ../antora/_site/keycloak-benchmark/dashbuilder
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v1
         with:

diff --git a/.github/workflows/keycloak-scalability-benchmark.yml b/.github/workflows/keycloak-scalability-benchmark.yml
@@ -82,7 +82,7 @@ jobs:
 
       - name: Build with Maven
         run: |
-          ./mvnw -B clean install -DskipTests
+          ./mvnw -B clean package -DskipTests -pl benchmark
           tar xfvz benchmark/target/keycloak-benchmark-*.tar.gz
           mv keycloak-benchmark-* keycloak-benchmark
 
@@ -94,14 +94,14 @@ jobs:
           disableStickySessions: true
 
       - name: Create Keycloak dataset with "${{ inputs.numberOfEntitiesInRealm }}" clients
-        if: ${{ !inputs.skipCreateDataset }} && inputs.scenarioName == 'authentication.ClientSecret'
+        if: "!inputs.skipCreateDataset && inputs.scenarioName == 'authentication.ClientSecret'"
         uses: ./.github/actions/keycloak-create-dataset
         with:
           project: ${{ env.PROJECT }}
           clients: ${{ inputs.numberOfEntitiesInRealm }}
 
       - name: Create Keycloak dataset with "${{ inputs.numberOfEntitiesInRealm }}" users
-        if: ${{ !inputs.skipCreateDataset }} && inputs.scenarioName == 'authentication.AuthorizationCode'
+        if: "!inputs.skipCreateDataset && inputs.scenarioName == 'authentication.AuthorizationCode'"
         uses: ./.github/actions/keycloak-create-dataset
         with:
           project: ${{ env.PROJECT }}
@@ -126,6 +126,7 @@ jobs:
 
       - name: Run "authentication.AuthorizationCode" scalability scenario
         if: inputs.scenarioName == 'authentication.AuthorizationCode'
+        id: kcb
         run: |
           bin/kcb.sh --scenario=keycloak.scenario."${{ inputs.scenarioName }}" \
             --server-url=${{ env.KEYCLOAK_URL }} \
@@ -143,8 +144,50 @@ jobs:
           path: keycloak-benchmark/results
           retention-days: 5
 
+      - name: Archive Summary
+        uses: actions/upload-artifact@v3
+        with:
+          name: summary
+          path: ${{ steps.kcb.outputs.kcb_result }}
+          retention-days: 5
+
       - name: Delete Keycloak deployment
         if: ${{ !inputs.skipDeleteProject }}
         uses: ./.github/actions/keycloak-delete-deployment
         with:
           project: ${{ env.PROJECT }}
+
+  archive:
+    name: Commit results to Git repository
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    needs:
+      - run
+    steps:
+
+      - name: Checkout repository for results
+        uses: actions/checkout@v3
+        with:
+          ref: 'result-data'
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: summary
+
+      - name: Commit result-summary
+        shell: bash
+        env:
+          GITHUB_OAUTH: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --global user.name "github-actions[bot]"
+          DATE_FOLDER=scalability/$(date +"%Y/%m/%d")
+          mkdir -p ${DATE_FOLDER}
+          mv *.json ${DATE_FOLDER}
+          git add .
+          git commit -m "generated"
+          git push
+
+
diff --git a/.github/workflows/rosa-run-benchmark.yml b/.github/workflows/rosa-run-benchmark.yml
@@ -64,7 +64,7 @@ jobs:
 
       - name: Build with Maven
         run: |
-          ./mvnw -B clean install -DskipTests
+          ./mvnw -B clean package -DskipTests -pl benchmark
           tar xfvz benchmark/target/keycloak-benchmark-*.tar.gz
           mv keycloak-benchmark-* keycloak-benchmark
 
@@ -87,6 +87,7 @@ jobs:
           project: ${{ env.PROJECT }}
 
       - name: Run Authorization Code scenario
+        id: kcb
         run: bin/kcb.sh --scenario=keycloak.scenario.authentication.AuthorizationCode --server-url=${{ env.KEYCLOAK_URL }} --realm-name=realm-0
         working-directory: keycloak-benchmark
 
@@ -97,8 +98,49 @@ jobs:
           path: keycloak-benchmark/results
           retention-days: 5
 
+      - name: Archive Summary
+        uses: actions/upload-artifact@v3
+        with:
+          name: summary
+          path: ${{ steps.kcb.outputs.kcb_result }}
+          retention-days: 5
+
       - name: Delete Keycloak deployment
         if: ${{ !inputs.skipDeleteProject }}
         uses: ./.github/actions/keycloak-delete-deployment
         with:
           project: ${{ env.PROJECT }}
+
+  archive:
+    name: Commit results to Git repository
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    needs:
+      - run
+    steps:
+
+      - name: Checkout repository for results
+        uses: actions/checkout@v3
+        with:
+          ref: 'result-data'
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: summary
+
+      - name: Commit result-summary
+        shell: bash
+        env:
+          GITHUB_OAUTH: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --global user.name "github-actions[bot]"
+          DATE_FOLDER=benchmark/$(date +"%Y/%m/%d")
+          mkdir -p ${DATE_FOLDER}
+          mv *.json ${DATE_FOLDER}
+          git add .
+          git commit -m "generated"
+          git push
+
diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml
@@ -21,6 +21,8 @@ jobs:
   build:
     name: Publish binaries
     runs-on: ubuntu-latest
+    # publishing can only happen on the main repository, not in forks
+    if: github.repository == 'keycloak/keycloak-benchmark'
 
     steps:
       - uses: actions/checkout@v3

diff --git a/benchmark/src/main/content/bin/kcb.sh b/benchmark/src/main/content/bin/kcb.sh
@@ -1,5 +1,4 @@
 #!/bin/bash
-
 case "$(uname)" in
     CYGWIN*)
         CFILE=$(cygpath "$0")
@@ -112,11 +111,50 @@ run_benchmark_with_workload() {
   local OUTPUT_DIR="${4:-"$DIRNAME/../results/"}"
   echo "INFO: Running benchmark with $1=$2, result output will be available in: $OUTPUT_DIR"
   mkdir -p "$OUTPUT_DIR"
+  DATE_START_UNIX=$(date +%s)
+  DATE_START_ISO=$(date --iso-8601=seconds)
+  DATE_START_ISO_COMPRESSED=$(date '+%Y%m%d-%H%M%S')
   if [ "$MODE" = "incremental" ]; then
     java $JAVA_OPTS "${SERVER_OPTS[@]}" "${CONFIG_ARGS[@]}" "-D$1=$2" "-Dmeasurement=${3:-30}" -cp $CLASSPATH_OPTS io.gatling.app.Gatling -bf $DIRNAME -rf "$OUTPUT_DIR" -s $SCENARIO > "$OUTPUT_DIR/gatling.log" 2>&1
   else
     java $JAVA_OPTS "${SERVER_OPTS[@]}" "${CONFIG_ARGS[@]}" "-D$1=$2" "-Dmeasurement=${3:-30}" -cp $CLASSPATH_OPTS io.gatling.app.Gatling -bf $DIRNAME -rf "$OUTPUT_DIR" -s $SCENARIO 2>&1 | tee "$OUTPUT_DIR/gatling.log"
   fi
+  EXIT_RESULT=$?
+  # don't include URLs or password information into the configuration recorded
+  CONFIG_ARGS_CLEAN=()
+  for index in "${!CONFIG_ARGS[@]}" ; do [[ "${CONFIG_ARGS[$index]}" =~ .*(url|pass).* ]] || CONFIG_ARGS_CLEAN+=(${CONFIG_ARGS[$index]}) ; done
+  OUTPUT_FOLDER=$OUTPUT_DIR/$(ls $OUTPUT_DIR -Art | grep -- -20 | tail -1)
+  DATE_END_UNIX=$(date +%s)
+  DATE_END_ISO=$(date --iso-8601=seconds)
+  jq '{ "grafana_output": { "stats": . } }' $OUTPUT_FOLDER/js/stats.json > $OUTPUT_FOLDER/result_grafana_stats.json
+  UUID=$(uuidgen)
+  jq '.'  > $OUTPUT_FOLDER/result_grafana_inputs.json <<EOF
+  {
+    "uuid": "${UUID}",
+    "name": "Scenario '${SCENARIO}' with $2 $1",
+    "grafana_input": {
+      "start": {
+        "epoch_seconds": ${DATE_START_UNIX},
+        "iso": "${DATE_START_ISO}"
+      },
+      "end": {
+        "epoch_seconds": ${DATE_END_UNIX},
+        "iso": "${DATE_END_ISO}"
+      },
+      "input": {
+        "scenario": "${SCENARIO}",
+        "unit": "$1",
+        "value": $2,
+        "config": "${CONFIG_ARGS_CLEAN[@]}"
+      }
+    }
+  }
+EOF
+  if [[ "${SUT_DESCRIBE}" != "" ]]; then
+    ${SUT_DESCRIBE} | jq '{ "system_under_test": . }' > ${OUTPUT_FOLDER}/result_sut.json
+  fi
+  jq -s add ${OUTPUT_FOLDER}/result_*.json > ${OUTPUT_FOLDER}/result-${DATE_START_ISO_COMPRESSED}-${UUID}.json
+  return ${EXIT_RESULT}
 }
 
 if [ "$MODE" = "incremental" ]; then
@@ -133,6 +171,7 @@ if [ "$MODE" = "incremental" ]; then
 
   RESULT_ROOT_DIR="$DIRNAME/../results/$MODE-$(date '+%Y%m%d%H%M%S')"
   mkdir -p $RESULT_ROOT_DIR
+  RESULT_ROOT_DIR=$(realpath ${RESULT_ROOT_DIR})
 
   #Incremental run is expected to do a warm up run to setup the system for the subsequent Incremental runs, you can ignore this run's result.
   echo "INFO: Running warm-up phase."
@@ -162,6 +201,11 @@ if [ "$MODE" = "incremental" ]; then
 
       echo "INFO: Last Successful workload for scenario $SCENARIO is $WORKLOAD_UNIT=$LAST_SUCCESSFUL_WORKLOAD."
       if [ $INCREMENT -eq 1 ]; then
+        ln -s $RESULT_ROOT_DIR/$WORKLOAD_UNIT-$LAST_SUCCESSFUL_WORKLOAD $RESULT_ROOT_DIR/last-successful
+        if [[ "${GITHUB_OUTPUT}" != "" ]]; then
+          OUTPUT_FOLDER=$RESULT_ROOT_DIR/last-successful/$(ls $RESULT_ROOT_DIR/last-successful -Art | grep -- -20 | tail -1)
+          echo "kcb_result=$OUTPUT_FOLDER/result-*.json" >> "${GITHUB_OUTPUT}"
+        fi
         echo "INFO: Reached the limit for scenario $SCENARIO with $WORKLOAD_UNIT=$LAST_SUCCESSFUL_WORKLOAD."
         exit
       fi
@@ -181,5 +225,10 @@ if [ "$MODE" = "incremental" ]; then
 else
   echo "INFO: Running benchmark in single-run mode."
   run_benchmark_with_workload $WORKLOAD_UNIT $CURRENT_WORKLOAD $MEASUREMENT
+  if [[ "${GITHUB_OUTPUT}" != "" ]]; then
+    OUTPUT_FOLDER=$DIRNAME/../results/$(ls $DIRNAME/../results -Art | grep -- -20 | tail -1)
+    OUTPUT_FOLDER=$(realpath ${OUTPUT_FOLDER})
+    echo "kcb_result=$OUTPUT_FOLDER/result-*.json" >> "${GITHUB_OUTPUT}"
+  fi
   exit
 fi
diff --git a/dashbuilder/.gitignore b/dashbuilder/.gitignore
@@ -0,0 +1,2 @@
+dist
+node_modules
diff --git a/dashbuilder/build.sh b/dashbuilder/build.sh
@@ -0,0 +1,3 @@
+#!/bin/bash -e
+yarn install
+yarn run build
diff --git a/dashbuilder/package.json b/dashbuilder/package.json
@@ -0,0 +1,17 @@
+{
+  "name": "dashbuilder-webapp",
+  "version": "0.0.0",
+  "description": "Dashbuilder WebApp",
+  "scripts": {
+    "bootstrap": "npm install",
+    "clean": "rm -rf dist/",
+    "copy:dashbuilder": "cp -r node_modules/@kie-tools/dashbuilder-client/dist/* dist",
+    "copy:sources": "cp -r static/* dist",
+    "build": "npm run clean && mkdir dist/ && npm run copy:dashbuilder && npm run copy:sources",
+    "server": "npm run build && cd dist && http-server -p 8000"
+  },
+  "devDependencies": {
+    "@kie-tools/dashbuilder-client": "^0.29.0",
+    "http-server": "^14.1.1"
+  }
+}