Skip to content

Queue Splitting Feature #11293

Queue Splitting Feature

Queue Splitting Feature #11293

Workflow file for this run

# Hello traveler!
# This is the CI workflow for mirrord.
# It is a bit complicated, but it is also very powerful.
# We try to optimize for speed, but sometimes there are limitations.
# Please try to document it here so people won't try repeating your errors.
# 1. GitHub cache is limited to 10GB, so we try to have less jobs to not exceed it, since if we get over 10GB
# the cache will be evicted then builds will be slower, so it's better to have less jobs.
# 2. I (Aviram) tried to use a container to build the Linux stuff, but couldn't get the e2e to work and (minikube in container)
# and the benefit seemed little.
# 3. Please be mindful, we try to target less than 30 minutes for the CI to run. In a perfect world it'd be less than 5m.
# If you're adding something, please make sure it doesn't impact too much, and if you're reviewing please have it in mind.
# 4. Make sure to specify target for cargo invocations, to re-use cache (cargo build --target X while host is X then cargo build will not use cache
# since it's different targets from it's perspective - https://doc.rust-lang.org/cargo/guide/build-cache.html
#
# - Adding a compiled app to e2e:
#
# If you want to add a rust/go/[other compiled language] to the list of e2e apps, then you have to check 2 other
# places.
#
# 1. The `test-images` repo, if your e2e wants to use a custom image, that's not already there, and;
# 2. The `mirrord-ci` repo, where you should add the compilation call to `e2e-setup-action/action.yaml`.
#
# Forgetting (2) will probably net you a message saying that `mirrord exec` could not find the path of the app,
# while forgetting (1) nets you a message saying that the service could not be reached/found.
#
# Caveats:
#
# You cannot use `workspace` dependencies in the test app, it fails to build, must fully specify the dependency.
#
# Related to (2): If you modify `mirrord-ci/actions.yaml`, you'll be affecting every PR!
name: CI
on:
workflow_dispatch:
push:
pull_request:
branches: [main]
types: [opened, synchronize, reopened, ready_for_review]
# Cancel previous runs on the same PR.
concurrency:
group: ${{ github.head_ref || github.run_id }}
cancel-in-progress: true
env:
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
MIRRORD_TELEMETRY: false
jobs:
towncrier_check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: install towncrier
run: pip install towncrier
- name: verify newsfragment exist
run: towncrier check
changed_files:
runs-on: ubuntu-latest
# don't run CI on drafts
if: github.event.pull_request.draft == false
outputs:
rs_changed: ${{ steps.changed-rs.outputs.any_changed }}
markdown_changed: ${{ steps.changed-markdown.outputs.any_changed }}
ci_changed: ${{ steps.changed-ci.outputs.any_changed }}
protocol_changed: ${{ steps.changed-protocol.outputs.any_changed }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: get CI changes
id: changed-ci
uses: tj-actions/changed-files@v39
with:
files: |
.github/workflows/ci.yaml
- name: get changed rs files
id: changed-rs
uses: tj-actions/changed-files@v39
with:
files: |
**/*.rs
mirrord/**
tests/**
Cargo.toml
Cargo.lock
.dockerignore
rust-toolchain.toml
rustfmt.toml
.cargo/**
- name: get markdown changes
id: changed-markdown
uses: tj-actions/changed-files@v39
with:
files: |
README.md
- name: get protocol changes
id: changed-protocol
uses: tj-actions/changed-files@v39
with:
files: |
mirrord/protocol/**
- name: get protocol toml changes
id: changed-protocol-toml
uses: tj-actions/changed-files@v39
with:
files: |
mirrord/protocol/Cargo.toml
- name: verify protocol bump
run: |
if [ "${{ steps.changed-protocol.outputs.any_changed }}" == "true" ] && [ "${{ steps.changed-protocol-toml.outputs.any_changed }}" != "true" ]; then
echo "Error: Protocol has changed but Cargo.toml has not. Please update Cargo.toml."
exit 1
fi
- name: output test
run: |
echo ${{ steps.changed-rs.outputs.any_changed }};
echo ${{ steps.changed-rs.outputs.all_changed_files }};
echo ${{ steps.changed-markdown.outputs.any_changed }};
echo ${{ steps.changed-markdown.outputs.all_changed_files }};
echo ${{ steps.changed-ci.outputs.any_changed }};
echo ${{ steps.changed-ci.outputs.all_changed_files }};
echo ${{ steps.changed-protocol.outputs.any_changed }};
echo ${{ steps.changed-protocol-toml.outputs.any_changed }};
lint:
runs-on: ubuntu-latest
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
steps:
- uses: actions/checkout@v3
- uses: arduino/setup-protoc@v1
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
# Otherwise the arguments to the setup-rust-toolchain action are ignored.
- run: rm rust-toolchain.toml
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: nightly-2024-04-15
components: rustfmt, clippy
target: aarch64-unknown-linux-gnu,x86_64-unknown-linux-gnu
- run: python3 -m pip install cargo-zigbuild
- run: cargo fmt --all -- --check
# x64
- run: cargo-zigbuild clippy --lib --bins --all-features --target x86_64-unknown-linux-gnu -- -Wclippy::indexing_slicing -D warnings
# Check that compiles for the supported linux targets (aarch64)
- run: cargo-zigbuild clippy --lib --bins --all-features --target aarch64-unknown-linux-gnu -- -Wclippy::indexing_slicing -D warnings
# if the branch is named is `x.x.x`, x ∈ [0, 9], then it's a release branch
# the output of this test is a boolean indicating if it's a release branch
# which is then used by `build_mirrord_on_release_branch`
check_if_release_branch:
runs-on: ubuntu-latest
outputs:
release_branch: ${{ steps.release-branch.outputs.branch }}
steps:
- id: release-branch
run: |
echo "branch=$([[ "${{ github.head_ref || github.ref_name }}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] && echo "true" || echo "false" )" >> "$GITHUB_OUTPUT"
- name: output test
run: |
echo ${{ steps.release-branch.outputs.branch }}
check-rust-docs:
runs-on: ubuntu-latest
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
env:
# enables the creation of a workspace index.html page.
RUSTDOCFLAGS: "--enable-index-page -Zunstable-options -Dwarnings"
steps:
- uses: actions/checkout@v3
- uses: arduino/setup-protoc@v1
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: nightly-2024-04-15
# TODO(alex): `no-deps` here due to an issue in `futures-util`.
- run: cargo doc --document-private-items --no-deps
test_agent:
runs-on: ubuntu-latest
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
container:
image: ghcr.io/metalbear-co/ci-agent-build:f8330d35a2a4b9132138f6fa9a3f3f80768c7c32
steps:
- uses: actions/checkout@v3
- uses: arduino/setup-protoc@v1
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: test
run: cargo test --target x86_64-unknown-linux-gnu -p mirrord-agent
test_agent_image:
runs-on: ubuntu-latest
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true' }}
steps:
- uses: actions/checkout@v3
- uses: docker/setup-buildx-action@v2
- name: build and export
uses: docker/build-push-action@v3
with:
context: .
tags: test
file: mirrord/agent/Dockerfile
outputs: type=docker,dest=/tmp/test.tar
cache-from: type=gha
cache-to: type=gha,mode=max
- name: upload image
uses: actions/upload-artifact@v3
with:
name: test
path: /tmp/test.tar
integration_tests:
runs-on: ubuntu-latest
needs: [changed_files]
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
steps:
- uses: actions/checkout@v3 # Checkout the mirrord repo.
- uses: actions-rust-lang/setup-rust-toolchain@v1 # Install rust.
with:
target: x86_64-unknown-linux-gnu
- run: |
cd mirrord/layer/tests/apps/issue1123
rustc issue1123.rs --out-dir target
- run: |
cd mirrord/layer/tests/apps/issue1054
rustc issue1054.rs --out-dir target
- run: |
cd mirrord/layer/tests/apps/issue1458
rustc issue1458.rs --out-dir target
- run: |
cd mirrord/layer/tests/apps/issue1458portnot53
rustc issue1458portnot53.rs --out-dir target
- run: |
cd mirrord/layer/tests/apps/issue2058
rustc issue2058.rs --out-dir target
- run: |
cd mirrord/layer/tests/apps/issue2204
rustc issue2204.rs --out-dir target
# For the `java_temurin_sip` test.
- uses: sdkman/sdkman-action@b1f9b696c79148b66d3d3a06f7ea801820318d0f
id: sdkman
with:
candidate: java
version: 17.0.6-tem
- run: java -version
- uses: actions/setup-node@v3 # For http mirroring test.
with:
node-version: 14
- run: npm install express # For http mirroring test with node.
- uses: actions/setup-python@v3 # For http mirroring tests with Flask and FastAPI.
- run: pip3 install flask fastapi uvicorn[standard] # For http mirroring test with Flask.
# don't use "cache" for other Gos since it will try to overwrite and have bad results.
- uses: actions/setup-go@v4
with:
go-version: "1.19"
cache-dependency-path: tests/go-e2e/go.sum
- run: |
go version
- run: | # Build Go test apps.
./scripts/build_go_apps.sh 19
- uses: actions/setup-go@v4
with:
go-version: "1.20"
cache: false
- run: |
go version
- run: | # Build Go test apps.
./scripts/build_go_apps.sh 20
- uses: actions/setup-go@v4
with:
go-version: "1.21"
cache: false
- run: |
go version
- run: | # Build Go test apps.
./scripts/build_go_apps.sh 21
- run: |
cd mirrord/layer/tests/apps/fileops
cargo build
- run: |
cd mirrord/layer/tests/apps/outgoing
cargo build
- run: |
cd mirrord/layer/tests/apps/recv_from
cargo build
- run: |
cd mirrord/layer/tests/apps/dns_resolve
cargo build
- run: |
cd mirrord/layer/tests/apps/listen_ports
cargo build
- run: |
cd mirrord/layer/tests/apps/issue1776
cargo build
- run: |
cd mirrord/layer/tests/apps/issue1776portnot53
cargo build
- run: |
cd mirrord/layer/tests/apps/issue1899
cargo build
- run: |
cd mirrord/layer/tests/apps/issue2001
cargo build
- run: |
cd mirrord/layer/tests/apps/issue2438
cargo build
- run: ./scripts/build_c_apps.sh
- run: cargo test --target x86_64-unknown-linux-gnu -p mirrord-layer
- name: mirrord protocol UT
run: cargo test --target x86_64-unknown-linux-gnu -p mirrord-protocol
- name: mirrord config UT
run: cargo test --target x86_64-unknown-linux-gnu -p mirrord-config
- name: mirrord kube UT
run: cargo test --target x86_64-unknown-linux-gnu -p mirrord-kube --all-features
macos_tests:
runs-on: macos-13
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
env:
MIRRORD_TEST_USE_EXISTING_LIB: ../../target/x86_64-apple-darwin/debug/libmirrord_layer.dylib
steps:
- uses: actions/checkout@v3 # Checkout the mirrord repo.
# the setup rust toolchain action ignores the input if file exists.. so remove it
- run: rm rust-toolchain.toml
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
components: rustfmt, clippy
target: aarch64-apple-darwin
toolchain: nightly-2024-04-15
- name: Install Protoc
uses: arduino/setup-protoc@v2
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: clippy x64
run: cargo clippy -p mirrord -p mirrord-layer -p mirrord-sip --target=x86_64-apple-darwin -- -Wclippy::indexing_slicing -D warnings
- name: clippy aarch64
run: cargo clippy -p mirrord -p mirrord-layer -p mirrord-sip --target=aarch64-apple-darwin -- -Wclippy::indexing_slicing -D warnings
- name: mirrord SIP UT
run: cargo test --target=x86_64-apple-darwin -p mirrord-sip
# prepare stuff needed for integration tests
- run: |
cd mirrord/layer/tests/apps/issue1123
rustc issue1123.rs --out-dir target
- run: |
cd mirrord/layer/tests/apps/issue1054
rustc issue1054.rs --out-dir target
- run: |
cd mirrord/layer/tests/apps/issue1458
rustc issue1458.rs --out-dir target
- run: |
cd mirrord/layer/tests/apps/issue1458portnot53
rustc issue1458portnot53.rs --out-dir target
- run: |
cd mirrord/layer/tests/apps/issue2058
rustc issue2058.rs --out-dir target
- uses: actions/setup-go@v4
with:
go-version: "1.19"
cache-dependency-path: tests/go-e2e/go.sum
- run: |
go version
# don't use "cache" for other Gos since it will try to overwrite and have bad results.
- run: | # Build Go test apps.
./scripts/build_go_apps.sh 19
- uses: actions/setup-go@v4
with:
go-version: "1.20"
cache: false
- run: |
go version
- run: | # Build Go test apps.
./scripts/build_go_apps.sh 20
- uses: actions/setup-go@v4
with:
go-version: "1.21"
cache: false
- run: |
go version
- run: | # Build Go test apps.
./scripts/build_go_apps.sh 21
- run: |
cd mirrord/layer/tests/apps/fileops
cargo build
- run: |
cd mirrord/layer/tests/apps/outgoing
cargo build
- run: |
cd mirrord/layer/tests/apps/recv_from
cargo build
- run: |
cd mirrord/layer/tests/apps/dns_resolve
cargo build
- run: |
cd mirrord/layer/tests/apps/issue1776
cargo build
- run: |
cd mirrord/layer/tests/apps/issue1776portnot53
cargo build
- run: |
cd mirrord/layer/tests/apps/issue1899
cargo build
- run: |
cd mirrord/layer/tests/apps/issue2001
cargo build
- run: |
cd mirrord/layer/tests/apps/issue2438
cargo build
- run: ./scripts/build_c_apps.sh
# For the `java_temurin_sip` test.
- uses: sdkman/sdkman-action@b1f9b696c79148b66d3d3a06f7ea801820318d0f
id: sdkman
with:
candidate: java
version: 17.0.6-tem
- run: java -version
- uses: actions/setup-python@v3 # For http mirroring tests with Flask and FastAPI.
- run: pip3 install flask # For http mirroring test with Flask.
- run: pip3 install fastapi # For http mirroring test with FastAPI.
- run: pip3 install uvicorn[standard] # For http mirroring test with FastAPI.
- uses: actions/setup-node@v3
with:
node-version: 18
- run: npm install express # For http mirroring test with node.
- run: cargo build --target=x86_64-apple-darwin -p mirrord-layer # Build layer lib. The tests load it into the apps.
- name: mirrord layer tests
run: cargo test --target=x86_64-apple-darwin -p mirrord-layer
e2e:
runs-on: ubuntu-latest
strategy:
matrix:
container-runtime: ["docker", "containerd"]
name: e2e
needs: [test_agent_image, changed_files]
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
env:
MIRRORD_AGENT_RUST_LOG: "warn,mirrord=debug"
steps:
- uses: actions/checkout@v3
- uses: actions-rust-lang/setup-rust-toolchain@v1 # Install Rust.
- uses: metalbear-co/ci/e2e-setup-action@main
with:
container-runtime: ${{matrix.container-runtime}}
- name: download image
uses: actions/download-artifact@v3
with:
name: test
path: /tmp
- run: minikube image load /tmp/test.tar
# run the cli tests only once, i.e. docker runtime only
- name: Run cli E2E tests
if: ${{ matrix.container-runtime == 'docker' }}
run: |
cargo test --target=x86_64-unknown-linux-gnu -p tests --no-default-features --features cli -- --test-threads=6 cli
# By running the test of the targetless agent first, we prove it works on an empty cluster without any pods.
- name: Run targetless E2E test
run: |
cargo test --target=x86_64-unknown-linux-gnu -p tests --no-default-features --features targetless -- --test-threads=6 targetless
- name: Run all E2E tests - docker
if: ${{ matrix.container-runtime == 'docker' }}
run: |
cargo test --target=x86_64-unknown-linux-gnu -p tests --no-default-features --features docker,job -- --test-threads=6
cargo test --target=x86_64-unknown-linux-gnu -p tests --no-default-features --features docker,ephemeral -- --test-threads=6
- name: Run all E2E tests - containerd
if: ${{ matrix.container-runtime == 'containerd' }}
run: |
cargo test --target=x86_64-unknown-linux-gnu -p tests --no-default-features --features job -- --test-threads=6
cargo test --target=x86_64-unknown-linux-gnu -p tests --no-default-features --features ephemeral -- --test-threads=6
- name: Collect logs
if: ${{ failure() }}
run: |
kubectl describe pods
docker exec minikube find /var/log/pods -print -exec cat {} \;
lint_markdown:
runs-on: ubuntu-latest
needs: changed_files
if: ${{needs.changed_files.outputs.markdown_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
steps:
- uses: actions/checkout@v3
- uses: avto-dev/markdown-lint@v1
with:
config: "markdownlint-config.json"
args: "README.md"
# we build mirrord to run ide tests, while it can take time for e2e/integration to finish,
# building concurrently can run faster the release ide tests
build_mirrord_on_release_branch:
runs-on: ubuntu-latest
name: build mirrord
needs: check_if_release_branch
if: ${{ needs.check_if_release_branch.outputs.release_branch == 'true' }}
steps:
- uses: actions/checkout@v3
- uses: arduino/setup-protoc@v1
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions-rust-lang/setup-rust-toolchain@v1
- run: cargo build --manifest-path=./Cargo.toml
- name: upload layer
uses: actions/upload-artifact@v3
with:
name: mirrord-artifacts
path: |
target/debug/mirrord
if-no-files-found: error
# depends on `build_mirrord_on_release_branch` which provides the binary on the current tag
# `build_mirrord_on_release_branch` depends on `check_if_release_branch`
# which checks if the branch is a release branch
intellij_e2e_on_release_branch:
# requires test_agent to have image to download
needs: [build_mirrord_on_release_branch, test_agent_image]
uses: metalbear-co/mirrord-intellij/.github/workflows/reusable_e2e.yaml@main
with:
mirrord_release_branch: true
vscode_e2e_on_release_branch:
# requires test_agent to have image to download
needs: [build_mirrord_on_release_branch, test_agent_image]
uses: metalbear-co/mirrord-vscode/.github/workflows/reusable_e2e.yaml@main
with:
mirrord_release_branch: true
# We need some "accummulation" job here because bors fails (timeouts) to
# listen on matrix builds.
# Hence, we have some kind of dummy here that bors can listen on
ci-success:
name: ci
# We want this to run even if some of the required jobs got skipped
if: always()
needs:
[
towncrier_check,
changed_files,
intellij_e2e_on_release_branch,
vscode_e2e_on_release_branch,
test_agent_image,
macos_tests,
integration_tests,
e2e,
test_agent,
lint,
lint_markdown,
check-rust-docs,
]
runs-on: ubuntu-latest
steps:
- name: CI succeeded
# We have to do it in the shell since if it's in the if condition
# then skipping is considered success by branch protection rules
env:
CI_SUCCESS: ${{ (needs.changed_files.result == 'success') &&
(needs.towncrier_check.result == 'success') &&
(needs.test_agent_image.result == 'success' || needs.test_agent_image.result == 'skipped') &&
(needs.macos_tests.result == 'success' || needs.macos_tests.result == 'skipped') &&
(needs.integration_tests.result == 'success' || needs.integration_tests.result == 'skipped') &&
(needs.e2e.result == 'success' || needs.e2e.result == 'skipped') &&
(needs.test_agent.result == 'success' || needs.test_agent.result == 'skipped') &&
(needs.lint.result == 'success' || needs.lint.result == 'skipped') &&
(needs.lint_markdown.result == 'success' || needs.lint_markdown.result == 'skipped') &&
(needs.intellij_e2e_on_release_branch.result == 'success' || needs.intellij_e2e_on_release_branch.result == 'skipped') &&
(needs.vscode_e2e_on_release_branch.result == 'success' || needs.vscode_e2e_on_release_branch.result == 'skipped') &&
(needs.check-rust-docs.result == 'success' || needs.check-rust-docs.result == 'skipped') }}
run: echo $CI_SUCCESS && if [ "$CI_SUCCESS" == "true" ]; then echo "SUCCESS" && exit 0; else echo "Failure" && exit 1; fi