diff --git a/.github/buildomat/jobs/build-and-test-linux.sh b/.github/buildomat/jobs/build-and-test-linux.sh new file mode 100644 index 00000000000..fb65dc7ba08 --- /dev/null +++ b/.github/buildomat/jobs/build-and-test-linux.sh @@ -0,0 +1,73 @@ +#!/bin/bash +#: +#: name = "build-and-test (ubuntu-20.04)" +#: variety = "basic" +#: target = "ubuntu-20.04" +#: rust_toolchain = "nightly-2022-04-27" +#: output_rules = [ +#: "/var/tmp/omicron_tmp/*", +#: "!/var/tmp/omicron_tmp/crdb-base*", +#: "!/var/tmp/omicron_tmp/rustc*", +#: ] +#: + +set -o errexit +set -o pipefail +set -o xtrace + +cargo --version +rustc --version + +# +# Set up a custom temporary directory within whatever one we were given so that +# we can check later whether we left detritus around. +# +TEST_TMPDIR='/var/tmp/omicron_tmp' +echo "tests will store output in $TEST_TMPDIR" >&2 +mkdir "$TEST_TMPDIR" + +# +# Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test +# suite. +# +export PATH="$PATH:$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" + +banner prerequisites +ptime -m bash ./tools/install_builder_prerequisites.sh -y + +# +# We build with: +# +# - RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings": disallow warnings +# in CI builds. This can result in breakage when the toolchain is +# updated, but that should only happen with a change to the repo, which +# gives us an opportunity to find and fix any newly-introduced warnings. +# +# - `--locked`: do not update Cargo.lock when building. Checking in +# Cargo.lock ensures that everyone is using the same dependencies and +# also gives us a record of which dependencies were used for each CI +# run. Building with `--locked` ensures that the checked-in Cargo.lock +# is up to date. +# +banner build +export RUSTFLAGS="-D warnings" +export RUSTDOCFLAGS="-D warnings" +export TMPDIR=$TEST_TMPDIR +ptime -m cargo +'nightly-2022-04-27' build --locked --all-targets --verbose + +# +# NOTE: We're using using the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid +# having to rebuild here. +# +banner test +ptime -m cargo +'nightly-2022-04-27' test --workspace --locked --verbose \ + --no-fail-fast + +# +# Make sure that we have left nothing around in $TEST_TMPDIR. The easiest way +# to check is to try to remove it with `rmdir`. +# +unset TMPDIR +echo "files in $TEST_TMPDIR (none expected on success):" >&2 +find "$TEST_TMPDIR" -ls +rmdir "$TEST_TMPDIR" diff --git a/.github/buildomat/jobs/build-and-test.sh b/.github/buildomat/jobs/build-and-test.sh index 50dc04a1f41..eb5c241b389 100644 --- a/.github/buildomat/jobs/build-and-test.sh +++ b/.github/buildomat/jobs/build-and-test.sh @@ -4,7 +4,11 @@ #: variety = "basic" #: target = "helios-latest" #: rust_toolchain = "nightly-2022-04-27" -#: output_rules = [] +#: output_rules = [ +#: "/var/tmp/omicron_tmp/*", +#: "!/var/tmp/omicron_tmp/crdb-base*", +#: "!/var/tmp/omicron_tmp/rustc*", +#: ] #: set -o errexit @@ -18,9 +22,9 @@ rustc --version # Set up a custom temporary directory within whatever one we were given so that # we can check later whether we left detritus around. # -TEST_TMPDIR="${TMPDIR:-/var/tmp}/omicron_tmp" -echo "tests will store output in $TEST_TMPDIR" -mkdir $TEST_TMPDIR +TEST_TMPDIR='/var/tmp/omicron_tmp' +echo "tests will store output in $TEST_TMPDIR" >&2 +mkdir "$TEST_TMPDIR" # # Put "./cockroachdb/bin" and "./clickhouse" on the PATH for the test @@ -56,13 +60,14 @@ ptime -m cargo +'nightly-2022-04-27' build --locked --all-targets --verbose # having to rebuild here. # banner test -ptime -m cargo +'nightly-2022-04-27' test --workspace --locked --verbose +ptime -m cargo +'nightly-2022-04-27' test --workspace --locked --verbose \ + --no-fail-fast # # Make sure that we have left nothing around in $TEST_TMPDIR. The easiest way # to check is to try to remove it with `rmdir`. # unset TMPDIR -echo "files in $TEST_TMPDIR (none expected on success):" -find $TEST_TMPDIR -ls -rmdir $TEST_TMPDIR +echo "files in $TEST_TMPDIR (none expected on success):" >&2 +find "$TEST_TMPDIR" -ls +rmdir "$TEST_TMPDIR" diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9d63cd8976f..7c71125c96e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -78,88 +78,3 @@ jobs: run: ./tools/install_builder_prerequisites.sh -y - name: Test build documentation run: RUSTDOCFLAGS="-Dwarnings" cargo doc - - build-and-test: - env: - OMICRON_TMP: /tmp/omicron_tmp - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ ubuntu-18.04 ] - steps: - # actions/checkout@v2 - - uses: actions/checkout@28c7f3d2b5162b5ddd3dfd9a45aa55eaf396478b - - uses: Swatinem/rust-cache@v1 - if: ${{ github.ref != 'refs/heads/main' }} - - name: Report cargo version - run: cargo --version - - name: Remove unnecessary software - run: | - echo "Disk space:" - df -h - - if [ -d "/usr/share/dotnet" ]; then - echo "Removing dotnet" - sudo rm -rf /usr/share/dotnet - fi - if [ -d "/usr/local/lib/android" ]; then - echo "Removing android" - sudo rm -rf /usr/local/lib/android - fi - if [ -d "/usr/local/.ghcup" ]; then - echo "Removing haskell" - sudo rm -rf /usr/local/.ghcup - fi - if [ -d "/opt/hostedtoolcache/CodeQL" ]; then - echo "Removing CodeQL" - sudo rm -rf /opt/hostedtoolcache/CodeQL - fi - if [ -d "/usr/share/swift" ]; then - echo "Removing swift" - sudo rm -rf /usr/share/swift - fi - - echo "Disk space:" - df -h - - name: Update PATH - run: echo "$PWD/out/cockroachdb/bin:$PWD/out/clickhouse" >> "$GITHUB_PATH" - - name: Install Pre-Requisites - run: ./tools/install_builder_prerequisites.sh -y - - name: Create temporary directory for test outputs - run: mkdir -p $OMICRON_TMP - - name: Build - # We build with: - # - RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings": disallow warnings - # in CI builds. This can result in breakage when the toolchain is - # updated, but that should only happen with a change to the repo, which - # gives us an opportunity to find and fix any newly-introduced warnings. - # - `--locked`: do not update Cargo.lock when building. Checking in - # Cargo.lock ensures that everyone is using the same dependencies and - # also gives us a record of which dependencies were used for each CI - # run. Building with `--locked` ensures that the checked-in Cargo.lock - # is up to date. - # - TMPDIR=$OMICRON_TMP: we specify a specific temporary directory so that - # failed test outputs will be in a known place that we can grab at the - # end without also grabbing random other temporary files. - run: TMPDIR=$OMICRON_TMP RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo build --locked --all-targets --verbose - - name: Run tests - # Use the same RUSTFLAGS and RUSTDOCFLAGS as above to avoid having to - # rebuild here. - run: TMPDIR=$OMICRON_TMP RUSTFLAGS="-D warnings" RUSTDOCFLAGS="-D warnings" cargo test --no-fail-fast --workspace --locked --verbose - - name: Archive results left by tests - # actions/upload-artifact@v2.3.1 - uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 - with: - name: failed_test_outputs_${{ runner.os }} - retention-days: 7 - path: | - ${{ env.OMICRON_TMP }} - !${{ env.OMICRON_TMP }}/crdb-base - !${{ env.OMICRON_TMP }}/rustc* - # Fail the build if successful tests leave detritus in $TMPDIR. The easiest - # way to check if the directory is empty is to try to remove it with - # `rmdir`. - - name: Remove temporary directory on success (if this fails, tests leaked files in TMPDIR) - if: ${{ success() }} - run: rmdir $OMICRON_TMP