Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Commit

Permalink
Simplified CI.
Browse files Browse the repository at this point in the history
  • Loading branch information
@jorgecarleitao
jorgecarleitao committed Sep 25, 2021
1 parent dce372d commit 0eae210
Showing 1 changed file with 66 additions and 223 deletions.
289 changes: 66 additions & 223 deletions .github/workflows/test.yml
View file
Expand Up @@ -3,167 +3,75 @@ name: Check and test
on: [push, pull_request]

jobs:
# build the library, a compilation step used by multiple steps below
linux-build-lib:
name: Build
runs-on: ubuntu-latest
container:
image: amd64/rust
steps:
- uses: actions/checkout@v2
- name: Cache Cargo
uses: actions/cache@v2
with:
path: /github/home/.cargo
# these represent dependencies downloaded by cargo
# and thus do not depend on the OS, arch nor rust version.
key: cargo-cache-
- name: Cache dependencies
uses: actions/cache@v2
with:
path: /github/home/target
# these represent compiled steps of both dependencies and arrow
# and thus are specific for a particular OS, arch and rust version.
key: ${{ runner.os }}-amd64-target-cache-stable
- name: Setup toolchain
run: |
rustup toolchain install stable
rustup default stable
rustup component add rustfmt
- name: Build
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
cargo build
# test the crate
linux-test:
name: Test
needs: [linux-build-lib]
ubuntu-latest:
name: Test full
runs-on: ubuntu-latest
container:
image: amd64/rust
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Cache Cargo
uses: actions/cache@v2
with:
path: /github/home/.cargo
# these represent dependencies downloaded by cargo
# and thus do not depend on the OS, arch nor rust version.
key: cargo-cache-
- name: Cache dependencies
uses: actions/cache@v2
with:
path: /github/home/target
# these represent compiled steps of both dependencies and arrow
# and thus are specific for a particular OS, arch and rust version.
key: ${{ runner.os }}-amd64-target-cache-stable
- name: Setup toolchain
run: |
rustup toolchain install stable
rustup default stable
rustup component add rustfmt
submodules: true # needed to test IPC, which are located in a submodule
- name: Install Rust
run: rustup update stable
- uses: Swatinem/rust-cache@v1
- name: Setup parquet files
run: |
run: |
apt update && apt install python3-pip python3-venv -y -q
python3 -m venv venv
venv/bin/pip install pip --upgrade
venv/bin/pip install pyarrow==3
venv/bin/python parquet_integration/write_parquet.py
- name: Run
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
cargo test --features full
run: cargo test --features full

windows-and-macos:
name: ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [windows-latest, macos-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
with:
submodules: true
# TODO: this won't cache anything, which is expensive. Setup this action
# with a OS-dependent path.
- name: Setup toolchain
run: |
rustup toolchain install stable
rustup default stable
rustup component add rustfmt
submodules: true # needed to test IPC, which are located in a submodule
- name: Install Rust
run: rustup update stable
- uses: Swatinem/rust-cache@v1
- name: Run
shell: bash
# no need to run the whole thing. Rust guarantees interoperability
run: ARROW2_IGNORE_PARQUET= cargo test --features full

clippy:
name: Clippy
needs: [linux-build-lib]
runs-on: ubuntu-latest
container:
image: amd64/rust
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Cache Cargo
uses: actions/cache@v2
with:
path: /github/home/.cargo
# these represent dependencies downloaded by cargo
# and thus do not depend on the OS, arch nor rust version.
key: cargo-cache-
- name: Cache dependencies
uses: actions/cache@v2
with:
path: /github/home/target
# these represent compiled steps of both dependencies and arrow
# and thus are specific for a particular OS, arch and rust version.
key: ${{ runner.os }}-amd64-target-cache-stable
- name: Setup toolchain
run: |
rustup toolchain install stable
rustup default stable
rustup component add rustfmt clippy
- name: Run
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
cargo clippy --features full
- name: Install Rust
run: rustup update stable
- uses: Swatinem/rust-cache@v1
- name: Install clippy
run: rustup component add clippy
- name: "clippy --all"
run: cargo clippy --all --tests

miri-checks:
name: Miri
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions-rs/toolchain@v1
with:
submodules: true
- uses: actions/cache@v2
with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-miri-${{ hashFiles('**/Cargo.lock') }}
- name: Setup toolchain
toolchain: nightly-2021-07-09
override: true
- uses: Swatinem/rust-cache@v1
- name: Install Miri
run: |
rustup toolchain install nightly-2021-07-03
rustup default nightly-2021-07-03
rustup component add rustfmt miri
rustup component add miri
cargo miri setup
- name: Run
env:
RUST_BACKTRACE: full
RUST_LOG: 'trace'
# --skip io: miri can't handle opening of files, so we skip those
run: |
cargo miri setup
cargo clean
cargo miri test --features full -- --skip io::parquet --skip io::ipc
run: cargo miri test --features full -- --skip io::parquet --skip io::ipc

coverage:
name: Coverage
Expand All @@ -172,138 +80,73 @@ jobs:
- uses: actions/checkout@v2
with:
submodules: true
- name: Cache Cargo
uses: actions/cache@v2
with:
path: /home/runner/.cargo
# this key is not equal because the user is different than on a container (runner vs github)
key: cargo-coverage-cache-
- name: Cache dependencies
uses: actions/cache@v2
with:
path: /home/runner/target
# this key is not equal because coverage uses different compilation flags.
key: ${{ runner.os }}-amd64-target-coverage-cache-stable-
- name: Setup parquet files
run: |
run: |
apt update && apt install python3-pip python3-venv -y -q
python3 -m venv venv
venv/bin/pip install pip --upgrade
venv/bin/pip install pyarrow==3
venv/bin/python parquet_integration/write_parquet.py
- uses: Swatinem/rust-cache@v1
- name: Install tarpaulin
run: cargo install cargo-tarpaulin
- name: Run coverage
run: |
export CARGO_HOME="/home/runner/.cargo"
export CARGO_TARGET_DIR="/home/runner/target"
# 2020-11-15: There is a cargo-tarpaulin regression in 0.17.0
# see https://github.com/xd009642/tarpaulin/issues/618
cargo install --version 0.16.0 cargo-tarpaulin
cargo tarpaulin --features full --out Xml
run: cargo tarpaulin --features full --out Xml
- name: Report coverage
continue-on-error: true
run: bash <(curl -s https://codecov.io/bash)

feature-compilation:
name: Feature coverage
needs: [linux-build-lib]
runs-on: ubuntu-latest
container:
image: amd64/rust
steps:
- uses: actions/checkout@v2
- name: Cache Cargo
uses: actions/cache@v2
with:
path: /github/home/.cargo
# these represent dependencies downloaded by cargo
# and thus do not depend on the OS, arch nor rust version.
key: cargo-cache-
- name: Cache dependencies
uses: actions/cache@v2
with:
path: /github/home/target
# these represent compiled steps of both dependencies and arrow
# and thus are specific for a particular OS, arch and rust version.
key: ${{ runner.os }}-amd64-target-cache-stable
- name: Setup toolchain
run: |
rustup toolchain install stable
rustup default stable
rustup component add rustfmt clippy
cargo install cargo-all-features
- name: Install Rust
run: rustup update stable
- name: Setup all features
run: cargo install cargo-all-features
- uses: Swatinem/rust-cache@v1
- name: Run
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
cargo check-all-features
run: cargo check-all-features

# test the arrow crate builds against wasm32 in stable rust
wasm32-build:
name: wasm32 build
cross:
name: cross
runs-on: ubuntu-latest
container:
image: amd64/rust
strategy:
matrix:
target:
- wasm32-unknown-unknown
- i686-unknown-linux-gnu
- powerpc-unknown-linux-gnu
- powerpc64-unknown-linux-gnu
- mips-unknown-linux-gnu
- arm-linux-androideabi
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Cache Cargo
uses: actions/cache@v2
- uses: actions-rs/toolchain@v1
with:
path: /github/home/.cargo
# these represent dependencies downloaded by cargo
# and thus do not depend on the OS, arch nor rust version.
key: cargo-cache-
- name: Cache dependencies
uses: actions/cache@v2
toolchain: nightly
target: ${{ matrix.target }}
override: true
- uses: Swatinem/rust-cache@v1
- uses: actions-rs/cargo@v1
with:
path: /github/home/target
key: ${{ runner.os }}-amd64-target-wasm32-cache-nightly
- name: Setup toolchain
run: |
rustup toolchain install nightly
rustup override set nightly
rustup component add rustfmt
rustup target add wasm32-unknown-unknown
- name: Build arrow crate
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
# no need
cargo build --no-default-features --features=merge_sort,io_ipc,io_csv,io_print,io_json,io_parquet --target wasm32-unknown-unknown
use-cross: true
command: check
args: --features=merge_sort,io_ipc,io_csv,io_print,io_json,io_parquet --target ${{ matrix.target }}

linux-simd-test:
name: SIMD
needs: [linux-build-lib]
runs-on: ubuntu-latest
container:
image: amd64/rust
steps:
- uses: actions/checkout@v2
- uses: actions-rs/toolchain@v1
with:
submodules: true
- name: Cache Cargo
uses: actions/cache@v2
with:
path: /github/home/.cargo
# these represent dependencies downloaded by cargo
# and thus do not depend on the OS, arch nor rust version.
key: cargo-simd-cache-
- name: Cache dependencies
uses: actions/cache@v2
with:
path: /github/home/target
# these represent compiled steps of both dependencies and arrow
# and thus are specific for a particular OS, arch and rust version.
key: ${{ runner.os }}-amd64-target-simd-cache-
- name: Setup toolchain
run: |
rustup toolchain install nightly
rustup default nightly
toolchain: nightly
override: true
- uses: Swatinem/rust-cache@v1
- name: Run
run: |
export CARGO_HOME="/github/home/.cargo"
export CARGO_TARGET_DIR="/github/home/target"
# no need to run over all features: simd only affects the core
cargo test --no-default-features --features compute,simd --lib
# no need to run over all features: simd only affects the core
run: cargo test --tests --no-default-features --features compute,simd

0 comments on commit 0eae210

Please sign in to comment.