Merge pull request #2 from Lmh-java/minghao/performance-regression-test

Issue checkstyle#14599: Build performance regression test

.ci/baseline_benchmark.txt

@@ -0,0 +1,19 @@
+Running benchmark 1/3...
+================== BENCHMARK RESULT #1 ==================
+Memory Usage: 1721102336 bytes
+Execution Time: 21.68 s
+============================================================
+Running benchmark 2/3...
+================== BENCHMARK RESULT #2 ==================
+Memory Usage: 1732280320 bytes
+Execution Time: 19.70 s
+============================================================
+Running benchmark 3/3...
+================== BENCHMARK RESULT #3 ==================
+Memory Usage: 1714171904 bytes
+Execution Time: 20.26 s
+============================================================
+===================== BENCHMARK SUMMARY ====================
+Average Memory Usage: 1722518186 bytes
+Average Execution Time: 20.54 s
+============================================================

.ci/benchmark-config.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0"?>
+<!DOCTYPE module PUBLIC "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+  "http://checkstyle.sourceforge.net/dtds/configuration_1_3.dtd">
+<module name="Checker">
+  <module name="TreeWalker"/>
+</module>

.ci/check-performance-regression.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+#set -x
+MAX_DIFFERENCE=10
+
+# parse benchmark result
+parse_benchmark_result() {
+  BENCHMARK_PATH=$1
+  MEMORY=$(awk '/Average Memory Usage:/ {print $4}' "$BENCHMARK_PATH")
+  EXECUTION_TIME=$(awk '/Average Execution Time:/ {print $4}' "$BENCHMARK_PATH")
+
+  local RESULT_ARRAY=($MEMORY $EXECUTION_TIME)
+  echo "${RESULT_ARRAY[@]}"
+}
+
+# compare baseline and patch benchmarks
+compare_results() {
+  # Calculate percentage difference for memory usage
+  MEMORY_DIFFERENCE=$(echo "scale=4; ((${PATCH[0]} - ${BASELINE[0]}) / ${BASELINE[0]}) * 100" | bc)
+  echo "Memory Usage Difference: $MEMORY_DIFFERENCE%"
+
+  # Calculate percentage difference for execution time
+  EXECUTION_TIME_DIFFERENCE=$(echo "scale=4; ((${PATCH[1]} - ${BASELINE[1]}) / ${BASELINE[1]}) * 100" | bc)
+  echo "Execution Time Difference: $EXECUTION_TIME_DIFFERENCE%"
+
+  # Check if differences exceed the maximum allowed difference
+  if (( $(echo "$MEMORY_DIFFERENCE > $MAX_DIFFERENCE" | bc -l) )) || (( $(echo "$EXECUTION_TIME_DIFFERENCE > $MAX_DIFFERENCE" | bc -l) )); then
+    echo "Differences exceed the maximum allowed difference (${MAX_DIFFERENCE}%)!"
+    exit 1
+  else
+    echo "Differences are within the maximum allowed difference."
+    exit 0
+  fi
+}
+
+# parse baseline benchmark
+BASELINE=($(parse_benchmark_result "./.ci/baseline_benchmark.txt"))
+
+# package patch
+JAR_VERSION=$(grep -oP '<version>\K[^<]+' pom.xml)
+export MAVEN_OPTS='-Xmx2000m'
+mvn -e --no-transfer-progress -Passembly,no-validations package
+
+# run benchmark and parse result
+sh ./.ci/run-benchmark.sh "./target/checkstyle-${JAR_VERSION}-all.jar"
+PATCH=($(parse_benchmark_result "./patch_benchmark.txt"))
+
+# compare two metrics
+compare_results

.ci/run-benchmark.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# Check if jar path is provided
+JAR_PATH=$1
+if [ -z "$JAR_PATH" ]; then
+    echo "Usage: $0 <target jar path>"
+    exit 1
+fi
+
+# Sample project path
+SAMPLE_PROJECT="./.ci/jdk21"
+
+# run a command and time it
+time_command() {
+  # Run the command with time
+  TEST_COMMAND=$1
+  TIME_OUTPUT=$(/usr/bin/time -l $TEST_COMMAND 2>&1)
+  # Extract memory usage and execution time
+  MEMORY=$(echo "$TIME_OUTPUT" | awk '/maximum resident set size/ {print $1}')
+  EXECUTION_TIME=$(echo "$TIME_OUTPUT" | awk '/real/ {print $1}')
+
+  local RESULT_ARRAY=($MEMORY $EXECUTION_TIME)
+  echo "${RESULT_ARRAY[@]}"
+}
+
+# run the benchmark
+#run_benchmark() {
+#  local BENCHMARK=($(time_command "java -jar $JAR_PATH -c default_config.xml $SAMPLE_PROJECT"))
+#  echo "===================== BENCHMARK RESULT ====================="
+#  echo "Memory Usage: ${BENCHMARK[0]} bytes"
+#  echo "Execution Time: ${BENCHMARK[1]} s"
+#  echo "============================================================"
+#}
+run_benchmark() {
+  local TOTAL_MEMORY=0
+  local TOTAL_TIME=0
+  local NUM_RUNS=3
+
+  for ((i = 1; i <= NUM_RUNS; i++)); do
+    echo "Running benchmark ${i}/${NUM_RUNS}..."
+    local BENCHMARK=($(time_command "java -jar $JAR_PATH -c ./.ci/benchmark-config.xml $SAMPLE_PROJECT"))
+    TOTAL_MEMORY=$((TOTAL_MEMORY + BENCHMARK[0]))
+    TOTAL_TIME=$(echo "$TOTAL_TIME + ${BENCHMARK[1]}" | bc)
+    echo "================== BENCHMARK RESULT #${i} =================="
+    echo "Memory Usage: ${BENCHMARK[0]} bytes"
+    echo "Execution Time: ${BENCHMARK[1]} s"
+    echo "============================================================"
+  done
+
+  local AVG_MEMORY=$((TOTAL_MEMORY / NUM_RUNS))
+  local AVG_TIME=$(echo "scale=2; $TOTAL_TIME / $NUM_RUNS" | bc)
+
+  echo "===================== BENCHMARK SUMMARY ===================="
+  echo "Average Memory Usage: ${AVG_MEMORY} bytes"
+  echo "Average Execution Time: ${AVG_TIME} s"
+  echo "============================================================"
+}
+
+# save the benchmark result
+run_benchmark | tee ./patch_benchmark.txt
+

.github/workflows/check-performance-regression.yml

@@ -0,0 +1,53 @@
+#####################################################################################
+# GitHub Action to generate Checkstyle report.
+#
+# Workflow starts when:
+# 1) issue comment - created
+#
+# For another bucket, you will need to change the secrets.
+#####################################################################################
+name: Check-Performance-Regression
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches: '*'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: 11
+          distribution: 'temurin'
+
+      - name: Checkout patch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          path: ./checkstyle
+
+      - name: Clone JDK 21 Repo
+        uses: actions/checkout@v4
+        with:
+          repository: openjdk/jdk21
+          path: ./checkstyle/.ci/jdk21
+
+      - name: Setup local maven cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: checkstyle-maven-cache-${{ hashFiles('**/pom.xml') }}
+
+      - name: Run performance test
+        run: |
+          cd checkstyle
+          ./.ci/check-performance-regression.sh