diff --git a/.ci/check-performance-regression.sh b/.ci/check-performance-regression.sh new file mode 100755 index 000000000000..480de2f9ea62 --- /dev/null +++ b/.ci/check-performance-regression.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +set -e + +# max difference tolerance in % +MAX_DIFFERENCE=10 +# baseline of execution time in second +EXECUTION_TIME_BASELINE=415.72 + +# sample project path +SAMPLE_PROJECT="./.ci/jdk17" +CONFIG_FILE="./config/benchmark-config.xml" + +# run a command and time it +# $TEST_COMMAND: command being timed +time_command() { + # Run the command with time + local TIME_OUTPUT=$(command time -p "$@" 2>&1) + # Extract execution time + local EXECUTION_TIME=$(echo "$TIME_OUTPUT" | awk '/real/ {print $2}') + + echo "${EXECUTION_TIME}" +} + +# run the benchmark a few times to calculate the average metrics +# $JAR_PATH: path of the jar file being benchmarked +run_benchmark() { + local JAR_PATH=$1 + if [ -z "$JAR_PATH" ]; then + echo "Missing JAR_PATH as an argument." + exit 1 + fi + + local TOTAL_TIME=0 + local NUM_RUNS=3 + + [ ! -d "$SAMPLE_PROJECT" ] && + echo "Directory $SAMPLE_PROJECT DOES NOT exists." | exit 1 + + for ((i = 1; i <= NUM_RUNS; i++)); do + local CMD=(java -jar "$JAR_PATH" -c "$CONFIG_FILE" \ + -x .git -x module-info.java "$SAMPLE_PROJECT") + local BENCHMARK=($(time_command "${CMD[@]}")) + TOTAL_TIME=$(echo "$TOTAL_TIME + ${BENCHMARK}" | bc) + done + + # average execution time in patch + local AVG_TIME=$(echo "scale=2; $TOTAL_TIME / $NUM_RUNS" | bc) + echo "$AVG_TIME" +} + +# compare baseline and patch benchmarks +# $EXECUTION_TIME execution time of the patch +compare_results() { + local EXECUTION_TIME=$1 + if [ -z "$EXECUTION_TIME" ]; then + echo "Missing EXECUTION_TIME as an argument." + exit 1 + fi + # Calculate percentage difference for execution time + local EXECUTION_TIME_DIFFERENCE=$(echo "scale=4; \ + ((${EXECUTION_TIME} - ${EXECUTION_TIME_BASELINE}) / ${EXECUTION_TIME_BASELINE}) * 100" | bc) + echo "Execution Time Difference: $EXECUTION_TIME_DIFFERENCE%" + + # Check if differences exceed the maximum allowed difference + if (( $(echo "$EXECUTION_TIME_DIFFERENCE > $MAX_DIFFERENCE" | bc -l) )); then + echo "Difference exceeds the maximum allowed difference (${EXECUTION_TIME_DIFFERENCE}% \ + > ${MAX_DIFFERENCE}%)!" + exit 1 + else + echo "Difference is within the maximum allowed difference (${EXECUTION_TIME_DIFFERENCE}% \ + <= ${MAX_DIFFERENCE}%)." + exit 0 + fi +} + +# package patch +export MAVEN_OPTS='-Xmx2000m' +mvn -e --no-transfer-progress -Passembly,no-validations package + +# run benchmark +echo "Benchmark launching..." +AVG_TIME="$(run_benchmark "$(find "./target/" -type f -name "checkstyle-*-all.jar")")" +echo "===================== BENCHMARK SUMMARY ====================" +echo "Execution Time Baseline: ${EXECUTION_TIME_BASELINE} s" +echo "Average Execution Time: ${AVG_TIME} s" +echo "============================================================" + +# compare result with baseline +compare_results "$AVG_TIME" +exit $? diff --git a/.github/workflows/check-performance-regression.yml b/.github/workflows/check-performance-regression.yml new file mode 100644 index 000000000000..3d491d21f2b3 --- /dev/null +++ b/.github/workflows/check-performance-regression.yml @@ -0,0 +1,53 @@ +##################################################################################### +# GitHub Action to test performance regression. +# +# Workflow starts when: +# 1) push to master +# 2) PR created or pushed +# +##################################################################################### +name: Check-Performance-Regression + +on: + push: + branches: + - master + pull_request: + branches: '*' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: 11 + distribution: 'temurin' + + - name: Checkout patch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: ./checkstyle + + - name: Clone JDK 17 Repo + uses: actions/checkout@v4 + with: + repository: openjdk/jdk17 + path: ./checkstyle/.ci/jdk17 + + - name: Setup local maven cache + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: checkstyle-maven-cache-${{ hashFiles('**/pom.xml') }} + + - name: Run performance test + run: | + cd checkstyle + bash ./.ci/check-performance-regression.sh diff --git a/config/benchmark-config.xml b/config/benchmark-config.xml new file mode 100644 index 000000000000..b8a7382d776c --- /dev/null +++ b/config/benchmark-config.xml @@ -0,0 +1,660 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/config/jsoref-spellchecker/whitelist.words b/config/jsoref-spellchecker/whitelist.words index 264b665312f6..57fe7342ce55 100644 --- a/config/jsoref-spellchecker/whitelist.words +++ b/config/jsoref-spellchecker/whitelist.words @@ -310,6 +310,7 @@ Dforce Dgpg Dhttp Diachenko +diags dirname Djacoco Dlinkcheck @@ -377,6 +378,7 @@ endline ENMI ent enum +EOI eol eot equalsavoidnull @@ -397,6 +399,7 @@ Exif explicitinitialization expr Externalizable +failover fallsthrough fallthrough fallthru @@ -542,6 +545,7 @@ hne homepage horizontalwhitespace Hostname +hotspot href htaccess htag @@ -602,6 +606,7 @@ interfaceistype interfacememberimpliedmodifier interfacetypeparametername interp +Intf intqueue invalidformat invalidinherit @@ -724,6 +729,7 @@ JText JToggle JTool jtree +jtreg junit junitpioneer jvm @@ -741,6 +747,7 @@ Kordas Kotlin lambdabodylength lambdaparametername +langtools LBRACK lcurly Leanback @@ -871,7 +878,6 @@ ncal ncss ndex needbraces -NEGS Nejmeh NEQ nestedfordepth @@ -1277,6 +1283,7 @@ suppresswithplaintextcommentfilter suse svg svn +switchextra sxpath Sys sysextensions @@ -1339,6 +1346,7 @@ tta ttf tvf tw +Twr typecastparenpad typename typevariablenames