From 290d9faa612649e702c368f79a867d004961ff94 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 1 Apr 2024 18:31:28 -0700 Subject: [PATCH] Issue #14599: Create performance regression test CI --- .ci/baseline.benchmark | 15 ++++ .ci/check-performance-regression.sh | 48 +++++++++++++ .ci/run-benchmark.sh | 69 +++++++++++++++++++ .../check-performance-regression.yml | 53 ++++++++++++++ config/checkstyle-non-main-files-checks.xml | 2 +- 5 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 .ci/baseline.benchmark create mode 100755 .ci/check-performance-regression.sh create mode 100755 .ci/run-benchmark.sh create mode 100644 .github/workflows/check-performance-regression.yml diff --git a/.ci/baseline.benchmark b/.ci/baseline.benchmark new file mode 100644 index 000000000000..dfd49c42aef9 --- /dev/null +++ b/.ci/baseline.benchmark @@ -0,0 +1,15 @@ +Running benchmark 1/3... +================== BENCHMARK RESULT #1 ================== +Execution Time: 8.82 s +============================================================ +Running benchmark 2/3... +================== BENCHMARK RESULT #2 ================== +Execution Time: 8.37 s +============================================================ +Running benchmark 3/3... +================== BENCHMARK RESULT #3 ================== +Execution Time: 8.50 s +============================================================ +===================== BENCHMARK SUMMARY ==================== +Average Execution Time: 8.56 s +============================================================ diff --git a/.ci/check-performance-regression.sh b/.ci/check-performance-regression.sh new file mode 100755 index 000000000000..c8d44ccf8c8b --- /dev/null +++ b/.ci/check-performance-regression.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +set -e + +# max % difference tolerance +MAX_DIFFERENCE=10 + +# parse benchmark result at the given path +parse_benchmark_result() { + BENCHMARK_PATH=$1 + EXECUTION_TIME=$(awk '/Average Execution Time:/ {print $4}' "$BENCHMARK_PATH") + + local RESULT_ARRAY=($EXECUTION_TIME) + echo "${RESULT_ARRAY[@]}" +} + +# compare baseline and patch benchmarks +compare_results() { + # Calculate percentage difference for execution time + EXECUTION_TIME_DIFFERENCE=$(echo "scale=4; \ + ((${PATCH[0]} - ${BASELINE[0]}) / ${BASELINE[0]}) * 100" | bc) + echo "Execution Time Difference: $EXECUTION_TIME_DIFFERENCE%" + + # Check if differences exceed the maximum allowed difference + if (( $(echo "$EXECUTION_TIME_DIFFERENCE > $MAX_DIFFERENCE" | bc -l) )); then + echo "Differences exceed the maximum allowed difference (${MAX_DIFFERENCE}%)!" + exit 1 + else + echo "Differences are within the maximum allowed difference." + exit 0 + fi +} + +# parse baseline benchmark +BASELINE=($(parse_benchmark_result "./.ci/baseline.benchmark")) + +# package patch +export MAVEN_OPTS='-Xmx2000m' +mvn -e --no-transfer-progress -Passembly,no-validations package + +# run benchmark and parse result +JAR_FILE=$(find "./target/" -type f -name "checkstyle-*-all.jar") +bash ./.ci/run-benchmark.sh "${JAR_FILE}" +PATCH=($(parse_benchmark_result "./patch.benchmark")) + +# compare two metrics +compare_results +exit $? diff --git a/.ci/run-benchmark.sh b/.ci/run-benchmark.sh new file mode 100755 index 000000000000..90ae6933d35b --- /dev/null +++ b/.ci/run-benchmark.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +set -e + +# check if jar path is provided +JAR_PATH=$1 +if [ -z "$JAR_PATH" ]; then + echo "Usage: $0 " + exit 1 +fi + +# Sample project path +SAMPLE_PROJECT="./.ci/jdk22" + +# generate benchmark config file +generate_config() { + local CONFIG=" + + + +" + echo "$CONFIG" > ./.ci/benchmark-config.xml +} + +# run a command and time it +time_command() { + # Run the command with time + TEST_COMMAND=$1 + # shellcheck disable=SC2086 + TIME_OUTPUT=$(command time -p $TEST_COMMAND 2>&1) + # Extract execution time + EXECUTION_TIME=$(echo "$TIME_OUTPUT" | awk '/real/ {print $2}') + + local RESULT_ARRAY=($EXECUTION_TIME) + echo "${RESULT_ARRAY[@]}" +} + +# run the benchmark a few times to calculate the average metrics +run_benchmark() { + local TOTAL_TIME=0 + local NUM_RUNS=3 + + [ ! -d "$SAMPLE_PROJECT" ] && + echo "Directory $SAMPLE_PROJECT DOES NOT exists." | exit 1 + + generate_config + for ((i = 1; i <= NUM_RUNS; i++)); do + echo "Running benchmark ${i}/${NUM_RUNS}..." + local CMD="java -jar $JAR_PATH -c ./.ci/benchmark-config.xml -x .git $SAMPLE_PROJECT" + local BENCHMARK=($(time_command "$CMD")) + TOTAL_TIME=$(echo "$TOTAL_TIME + ${BENCHMARK[0]}" | bc) + echo "================== BENCHMARK RESULT #${i} ==================" + echo "Execution Time: ${BENCHMARK[0]} s" + echo "============================================================" + done + + local AVG_TIME=$(echo "scale=2; $TOTAL_TIME / $NUM_RUNS" | bc) + + echo "===================== BENCHMARK SUMMARY ====================" + echo "Average Execution Time: ${AVG_TIME} s" + echo "============================================================" +} + +# save the benchmark result +run_benchmark | tee ./patch.benchmark +exit $? + diff --git a/.github/workflows/check-performance-regression.yml b/.github/workflows/check-performance-regression.yml new file mode 100644 index 000000000000..4759fee8accb --- /dev/null +++ b/.github/workflows/check-performance-regression.yml @@ -0,0 +1,53 @@ +##################################################################################### +# GitHub Action to test performance regression. +# +# Workflow starts when: +# 1) push to master +# 2) PR created or pushed +# +##################################################################################### +name: Check-Performance-Regression + +on: + push: + branches: + - master + pull_request: + branches: '*' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: 11 + distribution: 'temurin' + + - name: Checkout patch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: ./checkstyle + + - name: Clone JDK 22 Repo + uses: actions/checkout@v4 + with: + repository: openjdk/jdk22 + path: ./checkstyle/.ci/jdk22 + + - name: Setup local maven cache + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: checkstyle-maven-cache-${{ hashFiles('**/pom.xml') }} + + - name: Run performance test + run: | + cd checkstyle + bash ./.ci/check-performance-regression.sh diff --git a/config/checkstyle-non-main-files-checks.xml b/config/checkstyle-non-main-files-checks.xml index 921dd0b4514f..375aa00436a3 100644 --- a/config/checkstyle-non-main-files-checks.xml +++ b/config/checkstyle-non-main-files-checks.xml @@ -19,7 +19,7 @@ - +