From 52e361ab1b14c8380bffca965e52515cefdafa74 Mon Sep 17 00:00:00 2001
From: Ashley Scopes <73482956+ascopes@users.noreply.github.com>
Date: Sat, 21 Jan 2023 14:28:13 +0000
Subject: [PATCH 1/5] Speed up builds by uploading report tarball rather than
 individual files at the end.

Switch out GZIP with XZ and use the maximum compression factor.

Provide Maven default flags for JVM and build config
---
 .github/workflows/build.yml  | 123 +++++++++++++----------------------
 .github/workflows/deploy.yml |   5 +-
 .mvn/jvm.config              |   1 +
 .mvn/maven.config            |   1 +
 4 files changed, 47 insertions(+), 83 deletions(-)
 create mode 100644 .mvn/jvm.config
 create mode 100644 .mvn/maven.config

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 3c8070ab4..46a28e9dd 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -79,19 +79,7 @@ jobs:
 
       - name: Compile and run tests
         shell: bash
-        run: >-
-          ./mvnw 
-          -B 
-          -e
-          -T1C
-          -U
-          --no-transfer-progress
-          -Dcheckstyle.skip=true
-          -Dlicense.skip=true
-          -Dstyle.color=always
-          -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
-          ${{ inputs.maven-opts }}
-          clean verify
+        run: ./mvnw -B -U -Dcheckstyle.skip=true -Dlicense.skip=true clean verify
 
       - name: Annotate test reports with build environment info
         if: always()
@@ -113,8 +101,9 @@ jobs:
           set +f -x
           # Allow globs to not match anything without causing errors.
           shopt -s nullglob
-          
-          tar -czvf reports-${{ matrix.java-version }}-${{ matrix.os-name }}.tgz \
+  
+          # XZ with max compression, more efficient than using GZip.  
+          XZ_OPT=-9 tar -Jcvf reports-${{ matrix.java-version }}-${{ matrix.os-name }}.tar.xz \
               **/target/failsafe-reports/** \
               **/target/surefire-reports/** \
               **/target/site/jacoco/unit/jacoco*.xml \
@@ -124,9 +113,9 @@ jobs:
         uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: zzz-reports-${{ matrix.java-version }}-${{ matrix.os-name }}
+          name: temp-reports-${{ matrix.java-version }}-${{ matrix.os-name }}
           if-no-files-found: error
-          path: reports-${{ matrix.java-version }}-${{ matrix.os-name }}.tgz
+          path: reports-${{ matrix.java-version }}-${{ matrix.os-name }}.tar.xz
           retention-days: 1
 
   mutation-tests:
@@ -149,21 +138,7 @@ jobs:
       - name: Run mutation tests
         shell: bash
         # Do not run on multiple threads, the plugin does not support this properly.
-        run: >-
-          ./mvnw 
-          -B
-          -e
-          -T1
-          -U
-          --also-make
-          -pl java-compiler-testing
-          --no-transfer-progress
-          -P mutation
-          -Dstyle.color=always
-          -Dcheckstyle.skip=true
-          -Dlicense.skip=true
-          -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
-          test
+        run: ./mvnw -B -U -am -pl java-compiler-testing -P mutation -Dcheckstyle.skip=true -Dlicense.skip=true test
 
       # Compress first so that the collection job later takes far less time (order of a few minutes
       # or so). GitHub does not compress these until after the workflow finishes, meaning when
@@ -177,16 +152,16 @@ jobs:
           set +f -x
           # Allow globs to not match anything without causing errors.
           shopt -s nullglob
-          
-          tar -czvf reports-mutation-tests.tgz **/target/pit-reports/pit-reports/**
+          # XZ with max compression, more efficient than using GZip.
+          XZ_OPT=-9 tar -Jcvf reports-mutation-tests.tar.xz **/target/pit-reports/pit-reports/**
 
       - name: Stash reports tarball
         uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: zzz-reports-mutation
+          name: temp-reports-mutation
           if-no-files-found: error
-          path: reports-mutation-tests.tgz
+          path: reports-mutation-tests.tar.xz
           retention-days: 30
 
   publish-test-reports:
@@ -211,19 +186,16 @@ jobs:
       - name: Download stashed tarballs
         uses: actions/download-artifact@v3
         with:
-          path: 'artifacts/reports-*.tgz'
+          path: 'artifacts/reports-*.tar.xz'
 
       - name: Decompress stashed report tarballs
         shell: bash
         run: |-
-          find artifacts/ -name "reports-*.tgz" -exec tar -xzvf {} \;
-          find artifacts/ -name "reports-*.tgz" -exec rm -v {} \;
+          find artifacts/ -name "reports-*.tar.xz" -exec \
+              tar --no-same-owner --no-same-permissions -Jxvf {} \;
 
-      - name: Delete temporary artifacts
-        uses: geekyeggo/delete-artifact@v2
-        with:
-          name: 'zzz-reports-*'
-          failOnError: false
+          find artifacts/ -name "reports-*.tar.xz" -exec \
+              rm -v {} \;
 
       - name: Publish test results
         continue-on-error: true
@@ -261,25 +233,41 @@ jobs:
           chmod -v +x codecov
           ./codecov -v
           # TODO: upload unit test and integration test reports separately so that
-          # codecov processes it correctly. Need to hardwire the paths in here somehow.
+          # codecov processes it correctly. Need to hard-wire the paths in here somehow.
           #./codecov -c -F unit -v
           #./codecov -c -F integration -v
 
-      # Now we are finishing, GitHub will compress this properly.
+      # For now, we need to recompress everything to reupload it in one tarball. This
+      # is due to an issue with how GitHub Actions does not handle uploading large numbers
+      # of files in bulk, but is fine uploading one massive file.
+      - name: Recompress reports into one tarball
+        if: always()
+        run: >-
+          set +f -x
+          XZ_OPT=-9 tar -Jcvf test-reports.tar.xz \
+              **/target/pit-reports/pit-reports/**
+              **/target/failsafe-reports/** \
+              **/target/surefire-reports/** \
+              **/target/site/jacoco/unit/jacoco*.xml \
+              **/target/site/jacoco/int/jacoco*.xml
+
+      # TODO(ascopes): upload reports directly when GitHub fixes actions/upload-artifact
+      #   to not take an exponentially increasing amount of time based on the number of uploads.
       - name: Upload all reports in one archive
         uses: actions/upload-artifact@v3
         if: always()
         with:
           name: test-reports
           if-no-files-found: error
-          path: |-
-            **/target/pit-reports/pit-reports/**
-            **/target/failsafe-reports/**
-            **/target/surefire-reports/**
-            **/target/site/jacoco/unit/jacoco*.xml
-            **/target/site/jacoco/int/jacoco*.xml
+          path: test-reports.tar.xz
           retention-days: 30
 
+      - name: Delete temporary artifacts
+        uses: geekyeggo/delete-artifact@v2
+        with:
+          name: 'temp-reports-*'
+          failOnError: false
+
   formatting:
     name: Check formatting and licenses
     runs-on: ubuntu-22.04
@@ -300,19 +288,12 @@ jobs:
       - name: Run checks
         shell: bash
         run: >-
-          ./mvnw 
-          -B
-          -e
-          -T1C
-          -U
-          --no-transfer-progress
+          ./mvnw -B -U
           -DskipTests=true
-          -Dstyle.color=always
           -Dmaven.main.skip
           -Dmaven.jar.skip
           -Dmaven.resources.skip
           -Dmaven.test.skip
-          -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
           verify
 
   dependency-check:
@@ -336,21 +317,15 @@ jobs:
         shell: bash
         run: >-
           ./mvnw 
-          -B
-          -e
-          -T1C
-          -U
-          -Pdependency-check
-          --no-transfer-progress
+          -B -U
+          -P dependency-check
           -DskipTests=true
-          -Dstyle.color=always
           -Dcheckstyle.skip=true
           -Dlicense.skip=true
           -Dmaven.main.skip
           -Dmaven.jar.skip
           -Dmaven.resources.skip
           -Dmaven.test.skip
-          -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
           verify
 
       - name: Archive Dependency Scan reports
@@ -382,19 +357,10 @@ jobs:
       - name: Generate JavaDocs
         shell: bash
         run: >-
-          ./mvnw 
-          -B
-          -e
-          -T1C
-          -U
-          -pl java-compiler-testing
-          --also-make
-          --no-transfer-progress
+          ./mvnw  -B -U -am -pl java-compiler-testing
           -Dmaven.test.skip=true
           -Dcheckstyle.skip=true
           -Dlicense.skip=true
-          -Dstyle.color=always
-          -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
           clean compile javadoc:jar
 
       - name: Upload JavaDocs as a build artifact
@@ -424,4 +390,3 @@ jobs:
         with:
           name: github-pages
           failOnError: false
-
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index d6eee1dd9..f7a4a63e8 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -62,7 +62,7 @@ jobs:
           
           info "Determining release version to use (this may take a moment)..."
           if [[ '${{ inputs.version }}' == "" ]]; then
-            release_version="$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout | sed 's/-SNAPSHOT//g')"
+            release_version="$(./mvnw -B help:evaluate -Dexpression=project.version -q -DforceStdout | sed 's/-SNAPSHOT//g')"
           else
             release_version='${{ inputs.version }}'
           fi
@@ -83,14 +83,11 @@ jobs:
           run <<-SCRIPT          
             ./mvnw -B -e \
                 -Preleases \
-                --no-transfer-progress \
                 -Darguments='${build_args[@]}' \
                 -DdryRun='${{ inputs.dry-run }}' \
-                -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 \
                 -Dpassword='${{ secrets.GITHUB_TOKEN }}' \
                 -DreleaseVersion='${release_version}' \
                 -DsignTag=false \
-                -Dstyle.color=always \
                 -Dtag='v${release_version}' \
                 release:prepare release:perform
           SCRIPT
diff --git a/.mvn/jvm.config b/.mvn/jvm.config
new file mode 100644
index 000000000..fbc2b50e9
--- /dev/null
+++ b/.mvn/jvm.config
@@ -0,0 +1 @@
+-XX:+TieredCompilation -XX:TieredStopAtLevel=1
diff --git a/.mvn/maven.config b/.mvn/maven.config
new file mode 100644
index 000000000..bde8ad255
--- /dev/null
+++ b/.mvn/maven.config
@@ -0,0 +1 @@
+-e -T1C -Dstyle.color=always -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 --no-transfer-progress

From 761bd0af64082793bd08ab7e961e6d8bdc0299fc Mon Sep 17 00:00:00 2001
From: Ashley Scopes <73482956+ascopes@users.noreply.github.com>
Date: Sat, 21 Jan 2023 15:40:18 +0000
Subject: [PATCH 2/5] Attempt to prevent permission denied error for MacOS tar
 reports

---
 .github/workflows/build.yml | 44 +++++--------------------------------
 mvnw                        |  2 ++
 2 files changed, 7 insertions(+), 39 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 46a28e9dd..7007b78ed 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -113,10 +113,10 @@ jobs:
         uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: temp-reports-${{ matrix.java-version }}-${{ matrix.os-name }}
+          name: reports-${{ matrix.java-version }}-${{ matrix.os-name }}
           if-no-files-found: error
           path: reports-${{ matrix.java-version }}-${{ matrix.os-name }}.tar.xz
-          retention-days: 1
+          retention-days: 30
 
   mutation-tests:
     name: Run mutation tests
@@ -159,7 +159,7 @@ jobs:
         uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: temp-reports-mutation
+          name: reports-mutation-tests
           if-no-files-found: error
           path: reports-mutation-tests.tar.xz
           retention-days: 30
@@ -191,11 +191,8 @@ jobs:
       - name: Decompress stashed report tarballs
         shell: bash
         run: |-
-          find artifacts/ -name "reports-*.tar.xz" -exec \
-              tar --no-same-owner --no-same-permissions -Jxvf {} \;
-
-          find artifacts/ -name "reports-*.tar.xz" -exec \
-              rm -v {} \;
+          find artifacts/ -name "reports-*.tar.xz" -exec tar -Jxvf {} \;
+          find artifacts/ -name "reports-*.tar.xz" -exec rm -v {} \;
 
       - name: Publish test results
         continue-on-error: true
@@ -237,37 +234,6 @@ jobs:
           #./codecov -c -F unit -v
           #./codecov -c -F integration -v
 
-      # For now, we need to recompress everything to reupload it in one tarball. This
-      # is due to an issue with how GitHub Actions does not handle uploading large numbers
-      # of files in bulk, but is fine uploading one massive file.
-      - name: Recompress reports into one tarball
-        if: always()
-        run: >-
-          set +f -x
-          XZ_OPT=-9 tar -Jcvf test-reports.tar.xz \
-              **/target/pit-reports/pit-reports/**
-              **/target/failsafe-reports/** \
-              **/target/surefire-reports/** \
-              **/target/site/jacoco/unit/jacoco*.xml \
-              **/target/site/jacoco/int/jacoco*.xml
-
-      # TODO(ascopes): upload reports directly when GitHub fixes actions/upload-artifact
-      #   to not take an exponentially increasing amount of time based on the number of uploads.
-      - name: Upload all reports in one archive
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: test-reports
-          if-no-files-found: error
-          path: test-reports.tar.xz
-          retention-days: 30
-
-      - name: Delete temporary artifacts
-        uses: geekyeggo/delete-artifact@v2
-        with:
-          name: 'temp-reports-*'
-          failOnError: false
-
   formatting:
     name: Check formatting and licenses
     runs-on: ubuntu-22.04
diff --git a/mvnw b/mvnw
index 5643201c7..62eb40523 100755
--- a/mvnw
+++ b/mvnw
@@ -307,6 +307,8 @@ export MAVEN_CMD_LINE_ARGS
 
 WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
 
+set -x
+
 exec "$JAVACMD" \
   $MAVEN_OPTS \
   $MAVEN_DEBUG_OPTS \

From 53ac13870be307bd0e23abdf6c352a8bcc54e32e Mon Sep 17 00:00:00 2001
From: Ashley Scopes <73482956+ascopes@users.noreply.github.com>
Date: Sat, 21 Jan 2023 15:45:51 +0000
Subject: [PATCH 3/5] Add .gitattributes

---
 .gitattributes | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 .gitattributes

diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 000000000..3abe85786
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,4 @@
+# Needed to stop Git Bash in CI accidentally injecting \r into jvm.config and breaking builds.
+*         text=auto     eol=lf
+*.cmd     text=auto     eol=crlf
+*.jar     -text

From 99b82d0c314ab1b95a7a99a53731c3f474c1ffc1 Mon Sep 17 00:00:00 2001
From: Ashley Scopes <73482956+ascopes@users.noreply.github.com>
Date: Sat, 21 Jan 2023 16:47:15 +0000
Subject: [PATCH 4/5] Refactor prepare-test-outputs-for-merge to run XSLT in
 windowed batches

---
 scripts/prepare-test-outputs-for-merge.sh | 40 +++++++++++++++++------
 1 file changed, 30 insertions(+), 10 deletions(-)

diff --git a/scripts/prepare-test-outputs-for-merge.sh b/scripts/prepare-test-outputs-for-merge.sh
index 809336ff2..9019b411c 100755
--- a/scripts/prepare-test-outputs-for-merge.sh
+++ b/scripts/prepare-test-outputs-for-merge.sh
@@ -89,12 +89,12 @@ info "Generated XSLT script at ${surefire_prefix_xslt}"
 
 function find-all-surefire-reports {
   info "Discovering Surefire test reports"
-  find . -wholename '**/target/surefire-reports/TEST-*Test.xml' -print0 | xargs -0
+  find . -wholename '**/target/surefire-reports/TEST-*Test.xml' -print
 }
 
 function find-all-failsafe-reports {
   info "Discovering Failsafe test reports"
-  find . -wholename '**/target/failsafe-reports/TEST-*Test.xml' -print0 | xargs -0
+  find . -wholename '**/target/failsafe-reports/TEST-*Test.xml' -print
 }
 
 function find-all-jacoco-reports {
@@ -109,19 +109,39 @@ function find-all-jacoco-reports {
   fi
 }
 
+function xsltproc-surefire-report {
+  local prefix="${1}"
+  local xslt="${2}"
+  local input_report="${3}"
+  local output_report="${4}"
+
+  if ! run <<< "xsltproc --stringparam prefix '${prefix}' '${xslt}' '${input_report}' > '${output_report}'"; then
+    err "Error invoking xsltproc! Erroneous report was:"
+    dump "${input_report}"
+    return 2
+  fi
+
+  rm "${input_report}"
+}
+
 info "Updating test reports..."
 report_count=0
-for report in $(find-all-surefire-reports) $(find-all-failsafe-reports); do
+prefix="[Java-${ci_java_version}-${ci_os}]"
+concurrency="$(($(nproc) * 2))"
+
+while read -r report; do
   report_count="$((report_count+1))"
   new_report="${report/.xml/-java-${ci_java_version}-${ci_os}.xml}"
-  prefix="[Java-${ci_java_version}-${ci_os}]"
-  if ! run <<< "xsltproc --stringparam prefix '${prefix}' '${surefire_prefix_xslt}' '${report}' > '${new_report}'"; then
-    err "Error invoking xsltproc! Erroneous report was:"
-    dump "${report}"
-    exit 2
+  xsltproc-surefire-report "${prefix}" "${surefire_prefix_xslt}" "${report}" "${new_report}" &
+
+  # Wait if we have the max number of jobs in the window running (cpu-count * 4)
+  if [ "$((report_count % concurrency))" -eq 0 ]; then
+    wait < <(jobs -p)
+    info "Waited for up to ${concurrency} jobs to complete, will now continue..."
   fi
-  run <<< "rm '${report}'"
-done
+done < <(find-all-surefire-reports; find-all-failsafe-reports)
+wait < <(jobs -p)
+
 success "Updated ${report_count} test reports"
 
 info "Updating coverage reports..."

From e0c6028137066c76115ea4e7a8842b46ebac5c9d Mon Sep 17 00:00:00 2001
From: Ashley Scopes <73482956+ascopes@users.noreply.github.com>
Date: Sat, 21 Jan 2023 16:58:35 +0000
Subject: [PATCH 5/5] Increase xsltproc concurrency

---
 scripts/prepare-test-outputs-for-merge.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/prepare-test-outputs-for-merge.sh b/scripts/prepare-test-outputs-for-merge.sh
index 9019b411c..afc5f561d 100755
--- a/scripts/prepare-test-outputs-for-merge.sh
+++ b/scripts/prepare-test-outputs-for-merge.sh
@@ -127,7 +127,7 @@ function xsltproc-surefire-report {
 info "Updating test reports..."
 report_count=0
 prefix="[Java-${ci_java_version}-${ci_os}]"
-concurrency="$(($(nproc) * 2))"
+concurrency="$(($(nproc) * 4))"
 
 while read -r report; do
   report_count="$((report_count+1))"