diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87765f68541..0751eccd8c8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,167 +3,75 @@ name: Check and test on: [push, pull_request] jobs: - # build the library, a compilation step used by multiple steps below - linux-build-lib: - name: Build - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v2 - - name: Cache Cargo - uses: actions/cache@v2 - with: - path: /github/home/.cargo - # these represent dependencies downloaded by cargo - # and thus do not depend on the OS, arch nor rust version. - key: cargo-cache- - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: /github/home/target - # these represent compiled steps of both dependencies and arrow - # and thus are specific for a particular OS, arch and rust version. - key: ${{ runner.os }}-amd64-target-cache-stable - - name: Setup toolchain - run: | - rustup toolchain install stable - rustup default stable - rustup component add rustfmt - - name: Build - run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - cargo build - # test the crate - linux-test: - name: Test - needs: [linux-build-lib] + ubuntu-latest: + name: Test full runs-on: ubuntu-latest - container: - image: amd64/rust steps: - uses: actions/checkout@v2 with: - submodules: true - - name: Cache Cargo - uses: actions/cache@v2 - with: - path: /github/home/.cargo - # these represent dependencies downloaded by cargo - # and thus do not depend on the OS, arch nor rust version. - key: cargo-cache- - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: /github/home/target - # these represent compiled steps of both dependencies and arrow - # and thus are specific for a particular OS, arch and rust version. - key: ${{ runner.os }}-amd64-target-cache-stable - - name: Setup toolchain - run: | - rustup toolchain install stable - rustup default stable - rustup component add rustfmt + submodules: true # needed to test IPC, which are located in a submodule + - name: Install Rust + run: rustup update stable + - uses: Swatinem/rust-cache@v1 - name: Setup parquet files - run: | + run: | apt update && apt install python3-pip python3-venv -y -q python3 -m venv venv venv/bin/pip install pip --upgrade venv/bin/pip install pyarrow==3 venv/bin/python parquet_integration/write_parquet.py - name: Run - run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - cargo test --features full + run: cargo test --features full windows-and-macos: name: ${{ matrix.os }} - runs-on: ${{ matrix.os }} strategy: matrix: os: [windows-latest, macos-latest] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 with: - submodules: true - # TODO: this won't cache anything, which is expensive. Setup this action - # with a OS-dependent path. - - name: Setup toolchain - run: | - rustup toolchain install stable - rustup default stable - rustup component add rustfmt + submodules: true # needed to test IPC, which are located in a submodule + - name: Install Rust + run: rustup update stable + - uses: Swatinem/rust-cache@v1 - name: Run shell: bash - # no need to run the whole thing. Rust guarantees interoperability run: ARROW2_IGNORE_PARQUET= cargo test --features full clippy: name: Clippy - needs: [linux-build-lib] runs-on: ubuntu-latest - container: - image: amd64/rust steps: - uses: actions/checkout@v2 - with: - submodules: true - - name: Cache Cargo - uses: actions/cache@v2 - with: - path: /github/home/.cargo - # these represent dependencies downloaded by cargo - # and thus do not depend on the OS, arch nor rust version. - key: cargo-cache- - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: /github/home/target - # these represent compiled steps of both dependencies and arrow - # and thus are specific for a particular OS, arch and rust version. - key: ${{ runner.os }}-amd64-target-cache-stable - - name: Setup toolchain - run: | - rustup toolchain install stable - rustup default stable - rustup component add rustfmt clippy - - name: Run - run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - cargo clippy --features full + - name: Install Rust + run: rustup update stable + - uses: Swatinem/rust-cache@v1 + - name: Install clippy + run: rustup component add clippy + - name: "clippy --all" + run: cargo clippy --all --tests miri-checks: name: Miri runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 with: - submodules: true - - uses: actions/cache@v2 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-miri-${{ hashFiles('**/Cargo.lock') }} - - name: Setup toolchain + toolchain: nightly-2021-07-09 + override: true + - uses: Swatinem/rust-cache@v1 + - name: Install Miri run: | - rustup toolchain install nightly-2021-07-03 - rustup default nightly-2021-07-03 - rustup component add rustfmt miri + rustup component add miri + cargo miri setup + - name: Run - env: - RUST_BACKTRACE: full - RUST_LOG: 'trace' # --skip io: miri can't handle opening of files, so we skip those - run: | - cargo miri setup - cargo clean - cargo miri test --features full -- --skip io::parquet --skip io::ipc + run: cargo miri test --features full -- --skip io::parquet --skip io::ipc coverage: name: Coverage @@ -172,138 +80,73 @@ jobs: - uses: actions/checkout@v2 with: submodules: true - - name: Cache Cargo - uses: actions/cache@v2 - with: - path: /home/runner/.cargo - # this key is not equal because the user is different than on a container (runner vs github) - key: cargo-coverage-cache- - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: /home/runner/target - # this key is not equal because coverage uses different compilation flags. - key: ${{ runner.os }}-amd64-target-coverage-cache-stable- - name: Setup parquet files - run: | + run: | apt update && apt install python3-pip python3-venv -y -q python3 -m venv venv venv/bin/pip install pip --upgrade venv/bin/pip install pyarrow==3 venv/bin/python parquet_integration/write_parquet.py + - uses: Swatinem/rust-cache@v1 + - name: Install tarpaulin + run: cargo install cargo-tarpaulin - name: Run coverage - run: | - export CARGO_HOME="/home/runner/.cargo" - export CARGO_TARGET_DIR="/home/runner/target" - - # 2020-11-15: There is a cargo-tarpaulin regression in 0.17.0 - # see https://github.com/xd009642/tarpaulin/issues/618 - cargo install --version 0.16.0 cargo-tarpaulin - cargo tarpaulin --features full --out Xml + run: cargo tarpaulin --features full --out Xml - name: Report coverage continue-on-error: true run: bash <(curl -s https://codecov.io/bash) feature-compilation: name: Feature coverage - needs: [linux-build-lib] runs-on: ubuntu-latest - container: - image: amd64/rust steps: - uses: actions/checkout@v2 - - name: Cache Cargo - uses: actions/cache@v2 - with: - path: /github/home/.cargo - # these represent dependencies downloaded by cargo - # and thus do not depend on the OS, arch nor rust version. - key: cargo-cache- - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: /github/home/target - # these represent compiled steps of both dependencies and arrow - # and thus are specific for a particular OS, arch and rust version. - key: ${{ runner.os }}-amd64-target-cache-stable - - name: Setup toolchain - run: | - rustup toolchain install stable - rustup default stable - rustup component add rustfmt clippy - cargo install cargo-all-features + - name: Install Rust + run: rustup update stable + - name: Setup all features + run: cargo install cargo-all-features + - uses: Swatinem/rust-cache@v1 - name: Run - run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - cargo check-all-features + run: cargo check-all-features - # test the arrow crate builds against wasm32 in stable rust - wasm32-build: - name: wasm32 build + cross: + name: cross runs-on: ubuntu-latest - container: - image: amd64/rust + strategy: + matrix: + target: + - wasm32-unknown-unknown + - i686-unknown-linux-gnu + - powerpc-unknown-linux-gnu + - powerpc64-unknown-linux-gnu + - mips-unknown-linux-gnu + - arm-linux-androideabi steps: - uses: actions/checkout@v2 with: submodules: true - - name: Cache Cargo - uses: actions/cache@v2 + - uses: actions-rs/toolchain@v1 with: - path: /github/home/.cargo - # these represent dependencies downloaded by cargo - # and thus do not depend on the OS, arch nor rust version. - key: cargo-cache- - - name: Cache dependencies - uses: actions/cache@v2 + toolchain: nightly + target: ${{ matrix.target }} + override: true + - uses: Swatinem/rust-cache@v1 + - uses: actions-rs/cargo@v1 with: - path: /github/home/target - key: ${{ runner.os }}-amd64-target-wasm32-cache-nightly - - name: Setup toolchain - run: | - rustup toolchain install nightly - rustup override set nightly - rustup component add rustfmt - rustup target add wasm32-unknown-unknown - - name: Build arrow crate - run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - # no need - cargo build --no-default-features --features=merge_sort,io_ipc,io_csv,io_print,io_json,io_parquet --target wasm32-unknown-unknown + use-cross: true + command: check + args: --features=merge_sort,io_ipc,io_csv,io_print,io_json,io_parquet --target ${{ matrix.target }} linux-simd-test: name: SIMD - needs: [linux-build-lib] runs-on: ubuntu-latest - container: - image: amd64/rust steps: - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 with: - submodules: true - - name: Cache Cargo - uses: actions/cache@v2 - with: - path: /github/home/.cargo - # these represent dependencies downloaded by cargo - # and thus do not depend on the OS, arch nor rust version. - key: cargo-simd-cache- - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: /github/home/target - # these represent compiled steps of both dependencies and arrow - # and thus are specific for a particular OS, arch and rust version. - key: ${{ runner.os }}-amd64-target-simd-cache- - - name: Setup toolchain - run: | - rustup toolchain install nightly - rustup default nightly + toolchain: nightly + override: true + - uses: Swatinem/rust-cache@v1 - name: Run - run: | - export CARGO_HOME="/github/home/.cargo" - export CARGO_TARGET_DIR="/github/home/target" - # no need to run over all features: simd only affects the core - cargo test --no-default-features --features compute,simd --lib + # no need to run over all features: simd only affects the core + run: cargo test --tests --no-default-features --features compute,simd