From 238797eb843d29664a0aa3a00bccb7abd83049cc Mon Sep 17 00:00:00 2001 From: Ashley Scopes <73482956+ascopes@users.noreply.github.com> Date: Mon, 2 Jan 2023 12:16:11 +0000 Subject: [PATCH 1/2] Adjust shell scripts to make Sonatype Lift happy --- .github/workflows/build.yml | 4 +- scripts/common.sh | 35 ++++++-- scripts/prepare-test-outputs-for-merge.sh | 104 ++++++++++++++-------- scripts/update-licenses.sh | 45 ++++++---- 4 files changed, 123 insertions(+), 65 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 01d662718..a9ae725ab 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -195,8 +195,8 @@ jobs: shell: bash run: >- scripts/prepare-test-outputs-for-merge.sh - ${{ matrix.java-version }} - ${{ matrix.os-name }} + -j "${{ matrix.java-version }}" + -o "${{ matrix.os-name }}" - name: Archive test and coverage reports uses: actions/upload-artifact@v3 diff --git a/scripts/common.sh b/scripts/common.sh index 8bc707067..4f6c1d726 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # # Copyright (C) 2022 - 2023 Ashley Scopes # @@ -18,7 +19,8 @@ ### Common functions. ### -unset undefined > /dev/null 2>&1 || true +# We use undefined as a placeholder elsewhere. +unset undefined &> /dev/null || true function log() { printf "\033[3;37m$(date "+%H:%M:%S.%3N") \033[0;37m| \033[0;1;${1}m%s:\033[0;${1}m %s\033[0m\n" \ @@ -31,17 +33,38 @@ function warn() { log 33 WARNING "${@}"; } function info() { log 35 INFO "${@}"; } function in-path() { - command -v "${1}" > /dev/null 2>&1 + command -v "${1}" &> /dev/null return "${?}" } function run() { - # Run in subshell to enable correct argument requoting. + # Run in subshell to enable correct argument re-quoting. + local file + file="$(mktemp)" + + if [ "$#" -gt 0 ]; then + err "Function called incorrectly. Pass script to run via stdin rather than as arguments." + err "Invocation: <${*}>" + exit 1 + fi + + { + echo "#!/usr/bin/env bash" + echo "PS4=$'Running: \e[1;33m$ \e[0;3;33m'" + echo "set -euxo pipefail" + cat - + } >> "${file}" + + set +e /usr/bin/env bash \ - 1> >(while IFS=$"\r\n" read -r line; do log 36 "STDOUT" "${line}"; done) \ - 2> >(while IFS=$"\r\n" read -r line; do log 37 "STDERR" "${line}"; done) \ - <<< "PS4=$'Running: \e[1;33m$ \e[0;3;33m'; set -euxo pipefail; ${@}" + 1> >(while IFS=$'\r\n' read -r line; do log 36 "STDOUT" "${line}"; done) \ + 2> >(while IFS=$'\r\n' read -r line; do log 37 "STDERR" "${line}"; done) \ + "${file}" + local exit_code="${?}" + rm "${file}" + set -e # Wait for stdout and stderr to flush for ~10ms sleep 0.01 + return "${exit_code}" } diff --git a/scripts/prepare-test-outputs-for-merge.sh b/scripts/prepare-test-outputs-for-merge.sh index d8081c3ba..e4cae339f 100755 --- a/scripts/prepare-test-outputs-for-merge.sh +++ b/scripts/prepare-test-outputs-for-merge.sh @@ -20,14 +20,35 @@ ### that the test applies to, and to rename the jacoco.xml files to match the Java version in use. ### -set -o errexit -set -o nounset -set -o pipefail +set -euo pipefail +set -x source "$(dirname "${BASH_SOURCE[0]}")/common.sh" -ci_java_version="${1?Pass the Java version as the first argument to this script!}" -ci_os="${2?Pass the OS name as the second argument to this script!}" +function usage() { + echo "USAGE: ${BASH_SOURCE[0]} [-h] -j -o " + echo " -h Show this message and exit." + echo " -j Specify the Java version to use." + echo " -o Specify the OS name to use." +} + +ci_java_version="" +ci_os="" + +while getopts "hj:o:" opt; do + case "${opt}" in + h) usage; exit 0 ;; + j) ci_java_version="${OPTARG}" ;; + o) ci_os="${OPTARG}" ;; + ?|*) usage; exit 1 ;; + esac +done + +if [ -z "${ci_java_version}" ] || [ -z "${ci_os}" ]; then + err "Missing required arguments" + usage + exit 1 +fi info "Looking for xsltproc binary..." @@ -38,21 +59,21 @@ if [[ -z ${CI+undefined} ]] && in-path xsltproc; then info "xsltproc appears to be installed, and this is not a CI run" elif [[ "${OSTYPE}" = "darwin"* ]] && in-path brew; then info "Installing xsltproc from homebrew" - run "brew install libxslt" + run <<< "brew install libxslt" info "Giving the brew xsltproc binary precedence over the default MacOS one..." export PATH="/usr/local/opt/libxslt/bin:${PATH}" elif [[ "${OSTYPE}" =~ /win.*|mingw|msys|cygwin/ ]] && in-path choco; then info "Installing xsltproc from choco" - run "choco install xsltproc" + run <<< "choco install xsltproc" elif [[ "${OSTYPE}" = "linux"* ]] && in-path apt-get; then info "Installing xsltproc using apt-get" - run "sudo apt-get install -qy xsltproc" + run <<< "sudo apt-get install -qy xsltproc" elif [[ "${OSTYPE}" = "linux"* ]] && in-path dnf; then info "Installing xsltproc using dnf" - run "sudo dnf install -qy xsltproc" + run <<< "sudo dnf install -qy xsltproc" elif [[ "${OSTYPE}" = "linux"* ]] && in-path yum; then info "Installing xsltproc using yum" - run "sudo yum install -qy xsltproc" + run <<< "sudo yum install -qy xsltproc" else err "Cannot find xsltproc, nor can I find a suitable package manager to install it with." err "Please install xsltproc manually and then try again." @@ -60,35 +81,39 @@ else fi success "Found $(command -v xsltproc)" -run "xsltproc --version" +run <<< "xsltproc --version" info "Generating Surefire XSLT script..." surefire_prefix_xslt_dir="$(mktemp -d)" -trap 'run rm -Rf "${surefire_prefix_xslt_dir}"' EXIT SIGINT SIGTERM SIGQUIT +function tidy() { + run <<< "rm -Rf '${surefire_prefix_xslt_dir}'" +} + +trap 'tidy' EXIT SIGINT SIGTERM SIGQUIT surefire_prefix_xslt="${surefire_prefix_xslt_dir}/surefire.xslt" -sed 's/^ //g' > "${surefire_prefix_xslt}" <<'EOF' - - - - - - - - - - - - - - - - +cat > "${surefire_prefix_xslt}" <<'EOF' + + + + + + + + + + + + + + + + EOF info "Generated XSLT script at ${surefire_prefix_xslt}" @@ -118,21 +143,22 @@ function find-all-jacoco-reports { info "Updating test reports..." report_count=0 for report in $(find-all-surefire-reports) $(find-all-failsafe-reports); do - let report_count=report_count+1 - new_report=${report/.xml/-java-${ci_java_version}-${ci_os}.xml} + report_count="$((report_count+1))" + new_report="${report/.xml/-java-${ci_java_version}-${ci_os}.xml}" prefix="[Java-${ci_java_version}-${ci_os}]" - run "xsltproc --stringparam prefix '${prefix}' '${surefire_prefix_xslt}' '${report}' > '${new_report}'" - run "rm '${report}'" + run <<< "xsltproc --stringparam prefix '${prefix}' '${surefire_prefix_xslt}' '${report}' > '${new_report}'" + run <<< "rm '${report}'" done success "Updated ${report_count} test reports" info "Updating coverage reports..." jacoco_count=0 for jacoco_report in $(find-all-jacoco-reports); do - let jacoco_count=jacoco_count+1 + jacoco_count="$((jacoco_count+1))" new_jacoco_report="${jacoco_report/.xml/-java-${ci_java_version}-${ci_os}.xml}" - run "mv '${jacoco_report}' '${new_jacoco_report}'" + run <<< "mv '${jacoco_report}' '${new_jacoco_report}'" done + success "Updated ${jacoco_count} coverage reports" success "Processing completed." diff --git a/scripts/update-licenses.sh b/scripts/update-licenses.sh index c209f0bdb..bd1b52ac8 100755 --- a/scripts/update-licenses.sh +++ b/scripts/update-licenses.sh @@ -25,33 +25,42 @@ set -o pipefail source "$(dirname "${BASH_SOURCE[0]}")/common.sh" -if [ "$#" -gt 0 ] && ("${1}" = "--help" ] || [ "${1}" = "-h" ]); then - echo "USAGE: ${0} [-h | --help] [--debug]" - echo "Reapply license headers across this repository." - echo "" - echo " -h | --help Show this message and exit." - echo " --debug Show verbose Maven output." - echo - exit 0 -fi - cd "$(dirname "${BASH_SOURCE[0]}")/.." mvn_flags=(-B -e) -if [ "$#" -gt 0 ] && [ "${1}" = "--debug" ]; then - mvn_flags+=(-x) -else - mvn_flags+=() -fi +function usage() { + echo "USAGE: ${BASH_SOURCE[0]} [-d] [-h]" + echo " -d Show debugging output from Maven." + echo " -h Show this message and exit." +} + +function help() { + usage + exit 0 +} + +function unknown-option() { + err "Unknown option '${1}'" + usage + exit 1 +} + +while getopts "dh" opt; do + case "${opt}" in + d) mvn_flags+=(-x) ;; + h) help ;; + *) unknown-option "${opt}" ;; + esac +done info "Cleaning workspace..." -run "./mvnw --quiet ${mvn_flags[@]} clean" +run <<< "./mvnw --quiet ${mvn_flags[*]} clean" info "Reapplying licenses across workspace..." -run "./mvnw ${mvn_flags[@]} license:remove license:format" +run <<< "./mvnw ${mvn_flags[*]} license:remove license:format" info "Running verification (skipping tests)" -run "./mvnw ${mvn_flags[@]} verify -DskipTests" +run <<< "./mvnw ${mvn_flags[*]} verify -DskipTests" success "Completed!" From eeea8c709e3f8173d4644172b30da1e82314d0ef Mon Sep 17 00:00:00 2001 From: Ashley Scopes <73482956+ascopes@users.noreply.github.com> Date: Mon, 2 Jan 2023 12:23:44 +0000 Subject: [PATCH 2/2] Suppress irrelevant security warnings --- .../io/github/ascopes/jct/workspaces/impl/FileBuilderImpl.java | 1 + .../io/github/ascopes/jct/workspaces/impl/TempDirectoryImpl.java | 1 + 2 files changed, 2 insertions(+) diff --git a/java-compiler-testing/src/main/java/io/github/ascopes/jct/workspaces/impl/FileBuilderImpl.java b/java-compiler-testing/src/main/java/io/github/ascopes/jct/workspaces/impl/FileBuilderImpl.java index b4693aa03..a7d9bb393 100644 --- a/java-compiler-testing/src/main/java/io/github/ascopes/jct/workspaces/impl/FileBuilderImpl.java +++ b/java-compiler-testing/src/main/java/io/github/ascopes/jct/workspaces/impl/FileBuilderImpl.java @@ -122,6 +122,7 @@ public ManagedDirectory copiedFromFile(Path file) { } @Override + @SuppressWarnings("findsecbugs:URLCONNECTION_SSRF_FD") public ManagedDirectory copiedFromUrl(URL url) { return uncheckedIo(() -> createFile(url.openStream())); } diff --git a/java-compiler-testing/src/main/java/io/github/ascopes/jct/workspaces/impl/TempDirectoryImpl.java b/java-compiler-testing/src/main/java/io/github/ascopes/jct/workspaces/impl/TempDirectoryImpl.java index 0c71296c6..c1ce36498 100644 --- a/java-compiler-testing/src/main/java/io/github/ascopes/jct/workspaces/impl/TempDirectoryImpl.java +++ b/java-compiler-testing/src/main/java/io/github/ascopes/jct/workspaces/impl/TempDirectoryImpl.java @@ -91,6 +91,7 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx * @return the temporary directory. */ @CheckReturnValue + @SuppressWarnings("findsecbugs:PATH_TRAVERSAL_IN") public static TempDirectoryImpl newTempDirectory(String name) { // TODO(ascopes): are MS-DOS file name length limits a potential issue here? assertValidRootName(name);