From 183bafa3121ccac737c9352a583758b96c1914d7 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 1 Apr 2024 18:31:28 -0700 Subject: [PATCH] Issue #14599: Create performance regression test CI --- .ci/check-performance-regression.sh | 103 ++++ .../check-performance-regression.yml | 54 ++ config/benchmark-config.xml | 515 ++++++++++++++++++ 3 files changed, 672 insertions(+) create mode 100755 .ci/check-performance-regression.sh create mode 100644 .github/workflows/check-performance-regression.yml create mode 100644 config/benchmark-config.xml diff --git a/.ci/check-performance-regression.sh b/.ci/check-performance-regression.sh new file mode 100755 index 000000000000..d4f146a4f8dd --- /dev/null +++ b/.ci/check-performance-regression.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +set -e + +# max difference tolerance in % +THRESHOLD_PERCENTAGE=10 +# baseline of execution time in seconds +BASELINE_SECONDS=415.72 + +# JDK version +JDK_VERSION=17 +# sample project path +SAMPLE_PROJECT="./.ci-temp/jdk$JDK_VERSION" +# suppression file +SUPPRESSION_FILE="./config/projects-to-test/openjdk$JDK_VERSION-excluded.files" +# config file +CONFIG_FILE="./config/benchmark-config.xml" + +# execute a command and time it +# $TEST_COMMAND: command being timed +time_command() { + # execute the command and time it + /usr/bin/time -o time.temp -q -f "%e" "$@" &>result.tmp + + cat time.temp +} + +# execute the benchmark a few times to calculate the average metrics +# $JAR_PATH: path of the jar file being benchmarked +execute_benchmark() { + local JAR_PATH=$1 + if [ -z "$JAR_PATH" ]; then + echo "Missing JAR_PATH as an argument." + exit 1 + fi + + local TOTAL_SECONDS=0 + local NUM_EXECUTIONS=3 + + [ ! -d "$SAMPLE_PROJECT" ] && + echo "Directory $SAMPLE_PROJECT DOES NOT exist." | exit 1 + + # add suppressions to config file + sed -i "/ /r $SUPPRESSION_FILE" \ + $CONFIG_FILE + + for ((i = 0; i < NUM_EXECUTIONS; i++)); do + local CMD=(java -jar "$JAR_PATH" -c "$CONFIG_FILE" \ + -x .git -x module-info.java "$SAMPLE_PROJECT") + local BENCHMARK=($(time_command "${CMD[@]}")) + TOTAL_SECONDS=$(echo "$TOTAL_SECONDS + $BENCHMARK" | bc) + done + + # average execution time in patch + local AVERAGE_IN_SECONDS=$(echo "scale=2; $TOTAL_SECONDS / $NUM_EXECUTIONS" | bc) + echo "$AVERAGE_IN_SECONDS" +} + +# compare baseline and patch benchmarks +# $EXECUTION_TIME_SECONDS execution time of the patch +compare_results() { + local EXECUTION_TIME_SECONDS=$1 + if [ -z "$EXECUTION_TIME_SECONDS" ]; then + echo "Missing EXECUTION_TIME_SECONDS as an argument." + exit 1 + fi + # Calculate percentage difference for execution time + local DEVIATION_IN_SECONDS=$(echo "scale=4; \ + ((${EXECUTION_TIME_SECONDS} - ${BASELINE_SECONDS}) / ${BASELINE_SECONDS}) * 100" | bc) + echo "Execution Time Difference: $DEVIATION_IN_SECONDS%" + + # Check if differences exceed the maximum allowed difference + if (( $(echo "$DEVIATION_IN_SECONDS > $THRESHOLD_PERCENTAGE" | bc -l) )); then + echo "Difference exceeds the maximum allowed difference (${DEVIATION_IN_SECONDS}% \ + > ${THRESHOLD_PERCENTAGE}%)!" + exit 1 + else + echo "Difference is within the maximum allowed difference (${DEVIATION_IN_SECONDS}% \ + <= ${THRESHOLD_PERCENTAGE}%)." + exit 0 + fi +} + +# package patch +#mvn -e --no-transfer-progress -Passembly,no-validations package + +# start benchmark +echo "Benchmark launching..." +AVERAGE_IN_SECONDS="$(execute_benchmark "$(find "./target/" -type f -name "checkstyle-*-all.jar")")" + +# print the command execution result +echo "================ MOST RECENT COMMAND RESULT =================" +cat result.tmp + +echo "===================== BENCHMARK SUMMARY ====================" +echo "Execution Time Baseline: ${BASELINE_SECONDS} s" +echo "Average Execution Time: ${AVERAGE_IN_SECONDS} s" +echo "============================================================" + +# compare result with baseline +compare_results "$AVERAGE_IN_SECONDS" + +exit $? diff --git a/.github/workflows/check-performance-regression.yml b/.github/workflows/check-performance-regression.yml new file mode 100644 index 000000000000..bd0140ff0baf --- /dev/null +++ b/.github/workflows/check-performance-regression.yml @@ -0,0 +1,54 @@ +##################################################################################### +# GitHub Action to test performance regression. +# +# Workflow starts when: +# 1) push to master +# 2) PR created or pushed +# +##################################################################################### +name: Check-Performance-Regression + +on: + push: + branches: + - master + pull_request: + branches: '*' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + if: github.repository == 'checkstyle/checkstyle' + runs-on: ubuntu-latest + steps: + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: 11 + distribution: 'temurin' + + - name: Checkout patch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: ./checkstyle + + - name: Clone JDK 17 Repo + uses: actions/checkout@v4 + with: + repository: openjdk/jdk17 + path: ./checkstyle/.ci-temp/jdk17 + + - name: Setup local maven cache + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: checkstyle-maven-cache-${{ hashFiles('**/pom.xml') }} + + - name: Run performance test + run: | + cd checkstyle + bash ./.ci/check-performance-regression.sh diff --git a/config/benchmark-config.xml b/config/benchmark-config.xml new file mode 100644 index 000000000000..489e56b49609 --- /dev/null +++ b/config/benchmark-config.xml @@ -0,0 +1,515 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +