diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..3abe85786 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# Needed to stop Git Bash in CI accidentally injecting \r into jvm.config and breaking builds. +* text=auto eol=lf +*.cmd text=auto eol=crlf +*.jar -text diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3c8070ab4..7007b78ed 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -79,19 +79,7 @@ jobs: - name: Compile and run tests shell: bash - run: >- - ./mvnw - -B - -e - -T1C - -U - --no-transfer-progress - -Dcheckstyle.skip=true - -Dlicense.skip=true - -Dstyle.color=always - -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 - ${{ inputs.maven-opts }} - clean verify + run: ./mvnw -B -U -Dcheckstyle.skip=true -Dlicense.skip=true clean verify - name: Annotate test reports with build environment info if: always() @@ -113,8 +101,9 @@ jobs: set +f -x # Allow globs to not match anything without causing errors. shopt -s nullglob - - tar -czvf reports-${{ matrix.java-version }}-${{ matrix.os-name }}.tgz \ + + # XZ with max compression, more efficient than using GZip. + XZ_OPT=-9 tar -Jcvf reports-${{ matrix.java-version }}-${{ matrix.os-name }}.tar.xz \ **/target/failsafe-reports/** \ **/target/surefire-reports/** \ **/target/site/jacoco/unit/jacoco*.xml \ @@ -124,10 +113,10 @@ jobs: uses: actions/upload-artifact@v3 if: always() with: - name: zzz-reports-${{ matrix.java-version }}-${{ matrix.os-name }} + name: reports-${{ matrix.java-version }}-${{ matrix.os-name }} if-no-files-found: error - path: reports-${{ matrix.java-version }}-${{ matrix.os-name }}.tgz - retention-days: 1 + path: reports-${{ matrix.java-version }}-${{ matrix.os-name }}.tar.xz + retention-days: 30 mutation-tests: name: Run mutation tests @@ -149,21 +138,7 @@ jobs: - name: Run mutation tests shell: bash # Do not run on multiple threads, the plugin does not support this properly. - run: >- - ./mvnw - -B - -e - -T1 - -U - --also-make - -pl java-compiler-testing - --no-transfer-progress - -P mutation - -Dstyle.color=always - -Dcheckstyle.skip=true - -Dlicense.skip=true - -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 - test + run: ./mvnw -B -U -am -pl java-compiler-testing -P mutation -Dcheckstyle.skip=true -Dlicense.skip=true test # Compress first so that the collection job later takes far less time (order of a few minutes # or so). GitHub does not compress these until after the workflow finishes, meaning when @@ -177,16 +152,16 @@ jobs: set +f -x # Allow globs to not match anything without causing errors. shopt -s nullglob - - tar -czvf reports-mutation-tests.tgz **/target/pit-reports/pit-reports/** + # XZ with max compression, more efficient than using GZip. + XZ_OPT=-9 tar -Jcvf reports-mutation-tests.tar.xz **/target/pit-reports/pit-reports/** - name: Stash reports tarball uses: actions/upload-artifact@v3 if: always() with: - name: zzz-reports-mutation + name: reports-mutation-tests if-no-files-found: error - path: reports-mutation-tests.tgz + path: reports-mutation-tests.tar.xz retention-days: 30 publish-test-reports: @@ -211,19 +186,13 @@ jobs: - name: Download stashed tarballs uses: actions/download-artifact@v3 with: - path: 'artifacts/reports-*.tgz' + path: 'artifacts/reports-*.tar.xz' - name: Decompress stashed report tarballs shell: bash run: |- - find artifacts/ -name "reports-*.tgz" -exec tar -xzvf {} \; - find artifacts/ -name "reports-*.tgz" -exec rm -v {} \; - - - name: Delete temporary artifacts - uses: geekyeggo/delete-artifact@v2 - with: - name: 'zzz-reports-*' - failOnError: false + find artifacts/ -name "reports-*.tar.xz" -exec tar -Jxvf {} \; + find artifacts/ -name "reports-*.tar.xz" -exec rm -v {} \; - name: Publish test results continue-on-error: true @@ -261,25 +230,10 @@ jobs: chmod -v +x codecov ./codecov -v # TODO: upload unit test and integration test reports separately so that - # codecov processes it correctly. Need to hardwire the paths in here somehow. + # codecov processes it correctly. Need to hard-wire the paths in here somehow. #./codecov -c -F unit -v #./codecov -c -F integration -v - # Now we are finishing, GitHub will compress this properly. - - name: Upload all reports in one archive - uses: actions/upload-artifact@v3 - if: always() - with: - name: test-reports - if-no-files-found: error - path: |- - **/target/pit-reports/pit-reports/** - **/target/failsafe-reports/** - **/target/surefire-reports/** - **/target/site/jacoco/unit/jacoco*.xml - **/target/site/jacoco/int/jacoco*.xml - retention-days: 30 - formatting: name: Check formatting and licenses runs-on: ubuntu-22.04 @@ -300,19 +254,12 @@ jobs: - name: Run checks shell: bash run: >- - ./mvnw - -B - -e - -T1C - -U - --no-transfer-progress + ./mvnw -B -U -DskipTests=true - -Dstyle.color=always -Dmaven.main.skip -Dmaven.jar.skip -Dmaven.resources.skip -Dmaven.test.skip - -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 verify dependency-check: @@ -336,21 +283,15 @@ jobs: shell: bash run: >- ./mvnw - -B - -e - -T1C - -U - -Pdependency-check - --no-transfer-progress + -B -U + -P dependency-check -DskipTests=true - -Dstyle.color=always -Dcheckstyle.skip=true -Dlicense.skip=true -Dmaven.main.skip -Dmaven.jar.skip -Dmaven.resources.skip -Dmaven.test.skip - -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 verify - name: Archive Dependency Scan reports @@ -382,19 +323,10 @@ jobs: - name: Generate JavaDocs shell: bash run: >- - ./mvnw - -B - -e - -T1C - -U - -pl java-compiler-testing - --also-make - --no-transfer-progress + ./mvnw -B -U -am -pl java-compiler-testing -Dmaven.test.skip=true -Dcheckstyle.skip=true -Dlicense.skip=true - -Dstyle.color=always - -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 clean compile javadoc:jar - name: Upload JavaDocs as a build artifact @@ -424,4 +356,3 @@ jobs: with: name: github-pages failOnError: false - diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d6eee1dd9..f7a4a63e8 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -62,7 +62,7 @@ jobs: info "Determining release version to use (this may take a moment)..." if [[ '${{ inputs.version }}' == "" ]]; then - release_version="$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout | sed 's/-SNAPSHOT//g')" + release_version="$(./mvnw -B help:evaluate -Dexpression=project.version -q -DforceStdout | sed 's/-SNAPSHOT//g')" else release_version='${{ inputs.version }}' fi @@ -83,14 +83,11 @@ jobs: run <<-SCRIPT ./mvnw -B -e \ -Preleases \ - --no-transfer-progress \ -Darguments='${build_args[@]}' \ -DdryRun='${{ inputs.dry-run }}' \ - -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 \ -Dpassword='${{ secrets.GITHUB_TOKEN }}' \ -DreleaseVersion='${release_version}' \ -DsignTag=false \ - -Dstyle.color=always \ -Dtag='v${release_version}' \ release:prepare release:perform SCRIPT diff --git a/.mvn/jvm.config b/.mvn/jvm.config new file mode 100644 index 000000000..fbc2b50e9 --- /dev/null +++ b/.mvn/jvm.config @@ -0,0 +1 @@ +-XX:+TieredCompilation -XX:TieredStopAtLevel=1 diff --git a/.mvn/maven.config b/.mvn/maven.config new file mode 100644 index 000000000..bde8ad255 --- /dev/null +++ b/.mvn/maven.config @@ -0,0 +1 @@ +-e -T1C -Dstyle.color=always -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 --no-transfer-progress diff --git a/mvnw b/mvnw index 5643201c7..62eb40523 100755 --- a/mvnw +++ b/mvnw @@ -307,6 +307,8 @@ export MAVEN_CMD_LINE_ARGS WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain +set -x + exec "$JAVACMD" \ $MAVEN_OPTS \ $MAVEN_DEBUG_OPTS \ diff --git a/scripts/prepare-test-outputs-for-merge.sh b/scripts/prepare-test-outputs-for-merge.sh index 809336ff2..afc5f561d 100755 --- a/scripts/prepare-test-outputs-for-merge.sh +++ b/scripts/prepare-test-outputs-for-merge.sh @@ -89,12 +89,12 @@ info "Generated XSLT script at ${surefire_prefix_xslt}" function find-all-surefire-reports { info "Discovering Surefire test reports" - find . -wholename '**/target/surefire-reports/TEST-*Test.xml' -print0 | xargs -0 + find . -wholename '**/target/surefire-reports/TEST-*Test.xml' -print } function find-all-failsafe-reports { info "Discovering Failsafe test reports" - find . -wholename '**/target/failsafe-reports/TEST-*Test.xml' -print0 | xargs -0 + find . -wholename '**/target/failsafe-reports/TEST-*Test.xml' -print } function find-all-jacoco-reports { @@ -109,19 +109,39 @@ function find-all-jacoco-reports { fi } +function xsltproc-surefire-report { + local prefix="${1}" + local xslt="${2}" + local input_report="${3}" + local output_report="${4}" + + if ! run <<< "xsltproc --stringparam prefix '${prefix}' '${xslt}' '${input_report}' > '${output_report}'"; then + err "Error invoking xsltproc! Erroneous report was:" + dump "${input_report}" + return 2 + fi + + rm "${input_report}" +} + info "Updating test reports..." report_count=0 -for report in $(find-all-surefire-reports) $(find-all-failsafe-reports); do +prefix="[Java-${ci_java_version}-${ci_os}]" +concurrency="$(($(nproc) * 4))" + +while read -r report; do report_count="$((report_count+1))" new_report="${report/.xml/-java-${ci_java_version}-${ci_os}.xml}" - prefix="[Java-${ci_java_version}-${ci_os}]" - if ! run <<< "xsltproc --stringparam prefix '${prefix}' '${surefire_prefix_xslt}' '${report}' > '${new_report}'"; then - err "Error invoking xsltproc! Erroneous report was:" - dump "${report}" - exit 2 + xsltproc-surefire-report "${prefix}" "${surefire_prefix_xslt}" "${report}" "${new_report}" & + + # Wait if we have the max number of jobs in the window running (cpu-count * 4) + if [ "$((report_count % concurrency))" -eq 0 ]; then + wait < <(jobs -p) + info "Waited for up to ${concurrency} jobs to complete, will now continue..." fi - run <<< "rm '${report}'" -done +done < <(find-all-surefire-reports; find-all-failsafe-reports) +wait < <(jobs -p) + success "Updated ${report_count} test reports" info "Updating coverage reports..."