diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000000..2070b933fb2c --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,66 @@ +# Overview + +Our CI workflows make heavy usage of [Reusable Workflows](https://docs.github.com/en/actions/using-workflows/reusing-workflows). These reusable workflows can then be tested manually via the `manual_dispatch.yml` workflow. +Or integrated into CI jobs such has `on_pull_request.yml` or `on_main.yml`. + +By convention: +- All reusable workflows start with the `reusable_` prefix. +- All workflows that are triggered via `workflow_dispatch` start with the `manual_` prefix. +- All workflows that are triggered via an event start with the `on_` prefix. + - `on_pull_request` is triggered on pull requests. + - `on_main` is triggered on pushes to the main branch. + +If you are going to be doing any editing of workflows, the +[VS Code extension](https://marketplace.visualstudio.com/items?itemName=cschleiden.vscode-github-actions) +for GitHub Actions is highly recommended. + +## Reusable Workflows +- [reusable_checks.yml](reusable_checks.yml) - These are all the checks that run to ensure the code is formatted, + linted, and tested. This job produces no artifacts other than a pass/fail criteria for the build. + - `SAVE_CACHE` - If true, the rust cache will be saved. Generally we only do this for builds on `main` +- [reusable_bench.yml](reusable_bench.yml) - This job runs the benchmarks to check for performance regressions. + - `SAVE_BENCH` - If true, then the benchmark results are saved to update https://ref.rerun.io/dev/bench/ +- [reusable_deploy_docs](reusable_deploy_docs.yml) - This job deploys the python and rust documentation to https://ref.rerun.io + - `PY_DOCS_VERSION_NAME` - The name to use for versioning the python docs. This should generally match the version in + `Cargo.toml`. + - `UPDATE_LATEST` - If true, then the docs will be deployed to `latest/` as well as the versioned directory. +- [reusable_build_and_test_wheels.yml](reusable_build_and_test_wheels.yml) - This job builds the wheels, runs the +end-to-end test, and produces a sample RRD. The artifacts are accessible via GitHub artifacts, but not otherwise +uploaded anywhere. + - `MATURIN_FEATURE_FLAGS` - The feature flags to pass to maturin. + - `PLATFORM` - Which platform to build for: `linux`, `macos-arm`, `macos-intel`, or `windows`. + - `RELEASE_VERSION` - If producing a release, the version number. This must match the version in `Cargo.toml`. + - `RRD_ARTIFACT_NAME` - Intermediate name of the GitHub rrd artifact for passing to `reusable_upload_wheels.yml` + - `SAVE_CACHE` - If true, the rust cache will be saved. Generally we only do this for builds on `main` + - `WHEEL_ARTIFACT_NAME` - Intermediate name of the GitHub wheel artifact for passing to `reusable_upload_wheels.yml` +- [reusable_upload_wheels.yml](reusable_upload_wheels.yml) - This job uploads the wheels to google cloud + - `RRD_ARTIFACT_NAME` - Intermediate name of the GitHub rrd artifact. This should match the name passed to + `reusable_build_and_test_wheels.yml` + - `WHEEL_ARTIFACT_NAME` - Intermediate name of the GitHub wheel artifact. This should match the name passed to + `reusable_build_and_test_wheels.yml` +- [reusable_build_web.yml](reusable_build_web.yml) - This job builds the wasm artifacts for the web. + - `RELEASE_VERSION` - If producing a release, the version number. This must match the version in `Cargo.toml`. +- [reusable_upload_web.yml](reusable_upload_web.yml) - This job uploads the web assets to google cloud. By default this + only uploads to: `app.rerun.io/commit//` + - `MARK_PRERELEASE_FOR_MAINLINE` - If true, then the web assets will go to `app.rerun.io/preleease/ + - `MARK_TAGGED_VERSION` - If true, then the web assets will go to `app.rerun.io/version/` + - `RELEASE_VERSION` - If producing a release, the version number. + - `RRD_ARTIFACT_NAME` - Intermediate name of the GitHub rrd artifact. This should match the name passed to + `reusable_build_and_test_wheels.yml` + - `UPLOAD_COMMIT_OVERRIDE` - If set, will replace the value of ``. This is necessary because we want pull + request builds associated with their originating commit, even if the web-build happens on an ephemeral merge-commit. +- [reusable_pr_summary.yml](reusable_pr_summary.yml) - This job updates the PR summary with the results of the CI run. + - This summary can be found at: + `https://storage.googleapis.com/rerun-builds/pull_request//index.html` + - `PR_NUMBER` - The PR number to update. This will generally be set by the `on_pull_request.yml` workflow using: + `${{github.event.pull_request.number}}` + +## Manual Workflows +- [manual_dispatch](manual_dispatch.yml) - This workflow is used to manually trigger the assorted reusable workflows for + testing. + - See the workflow file for the list of parameters. +- [manual_build_wheels_for_pr.yml](manual_build_wheels_for_pr.yml) - This workflow can be dispatched on a branch and + will build all of the wheels for the associated pull-request. Uses: + - [reusable_build_and_test_wheels.yml](reusable_build_and_test_wheels.yml) + - [reusable_upload_wheels.yml](reusable_upload_wheels.yml) + - [reusable_pr_summary.yml](reusable_pr_summary.yml) diff --git a/.github/workflows/manual_build_wheels_for_pr.yml b/.github/workflows/manual_build_wheels_for_pr.yml index 9da53140b15b..162b41df1022 100644 --- a/.github/workflows/manual_build_wheels_for_pr.yml +++ b/.github/workflows/manual_build_wheels_for_pr.yml @@ -5,7 +5,7 @@ on: jobs: - check_for_pr: + check-for-pr: runs-on: ubuntu-latest outputs: PR_NUMBER: ${{ steps.get_pr.outputs.PR_NUMBER }} @@ -24,5 +24,89 @@ jobs: exit 1 else echo "Commit is associated with PR: $pr_number" - echo "PR_NUMBER=$pr_number" >> "$GITHUB_OUTPUTS" + echo "PR_NUMBER=$pr_number" >> "$GITHUB_OUTPUT" fi + + build-linux: + needs: [check-for-pr] + name: 'Linux: Build/Test Wheels' + uses: ./.github/workflows/reusable_build_and_test_wheels.yml + with: + PLATFORM: linux + WHEEL_ARTIFACT_NAME: linux-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + build-windows: + needs: [check-for-pr] + name: 'Windows: Build/Test Wheels' + uses: ./.github/workflows/reusable_build_and_test_wheels.yml + with: + PLATFORM: windows + WHEEL_ARTIFACT_NAME: windows-wheel + RRD_ARTIFACT_NAME: '' + secrets: inherit + + build-macos-arm: + needs: [check-for-pr] + name: 'Macos-Arm: Build/Test Wheels' + uses: ./.github/workflows/reusable_build_and_test_wheels.yml + with: + PLATFORM: macos-arm + WHEEL_ARTIFACT_NAME: macos-arm-wheel + RRD_ARTIFACT_NAME: '' + secrets: inherit + + build-macos-intel: + needs: [check-for-pr] + name: 'Macos-Intel: Build/Test Wheels' + uses: ./.github/workflows/reusable_build_and_test_wheels.yml + with: + PLATFORM: macos-intel + WHEEL_ARTIFACT_NAME: 'macos-intel-wheel' + RRD_ARTIFACT_NAME: '' + secrets: inherit + + upload-wheels-linux: + name: 'Linux: Upload Wheels' + needs: [build-linux] + uses: ./.github/workflows/reusable_upload_wheels.yml + with: + WHEEL_ARTIFACT_NAME: linux-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + upload-wheels-windows: + name: 'Windows: Upload Wheels' + needs: [build-linux, build-windows] + uses: ./.github/workflows/reusable_upload_wheels.yml + with: + WHEEL_ARTIFACT_NAME: windows-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + upload-wheels-macos-arm: + name: 'Macos-Arm: Upload Wheels' + needs: [build-linux, build-macos-arm] + uses: ./.github/workflows/reusable_upload_wheels.yml + with: + WHEEL_ARTIFACT_NAME: macos-arm-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + upload-wheels-macos-intel: + name: 'Macos-Intel: Upload Wheels' + needs: [build-linux, build-macos-intel] + uses: ./.github/workflows/reusable_upload_wheels.yml + with: + WHEEL_ARTIFACT_NAME: macos-intel-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + update-pr-summary: + name: 'Update PR Summary' + needs: [check-for-pr, upload-wheels-linux, upload-wheels-windows, upload-wheels-macos-arm, upload-wheels-macos-intel] + uses: ./.github/workflows/reusable_pr_summary.yml + with: + PR_NUMBER: ${{ needs.check-for-pr.outputs.PR_NUMBER}} + secrets: inherit diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml index edae126ebea4..6147c4013df1 100644 --- a/.github/workflows/manual_dispatch.yml +++ b/.github/workflows/manual_dispatch.yml @@ -5,96 +5,250 @@ on: # NOTE: boolean inputs are still actually strings # See: https://github.com/actions/runner/issues/1483 inputs: - LINTS: - description: 'Run Lints' + # Sorted based on job workflow logic rather than alphabetical + CHECKS: + description: 'Run reuseable_checks' type: boolean - required: true + required: false default: true - BUILD_LINUX: - description: 'Build/Test Linux' - type: boolean - required: true - default: true - BUILD_MACOS_INTEL: - description: 'Build/Test Mac (Intel)' + + DEPLOY_DOCS: + description: 'Run reusable_deploy_docs' type: boolean - required: true + required: false default: false - BUILD_MACOS_ARM: - description: 'Build/Test Mac (Arm)' + + BENCHES: + description: 'Run reusable_bench.yml' type: boolean - required: true + required: false default: false - BUILD_WINDOWS: - description: 'Build/Test Win' + + MIN_TEST_WHEEL: + description: 'Run reuseable_build_and_test_wheels (Minimal Wheel Linux Only)' type: boolean - required: true + required: false + + WHEEL_PLATFORMS: + description: 'Run reusable_build_and_test_wheels (Full build: linux,windows,macos-arm,macos-intel)' + type: string + required: false + default: 'linux' + + BUILD_WEB: + description: 'Run reusable_build_web' + type: boolean + required: false + default: true + + RELEASE_VERSION: + description: 'Release Version Number (Must match Cargo.toml)' + type: string + required: false + default: 'prerelease' + + UPLOAD_GCLOUD: + description: 'Upload wheels and wasm to gcloud' + type: boolean + required: false default: false - SAVE_CACHE: - description: 'Save Cache' + + UPDATE_PR_SUMMARY: + description: 'Update the PR Summary' type: boolean - required: true + required: false default: false - PACKAGE: - description: 'Package' + + SAVE_CACHE: + description: 'Save the rust-cache where relevant' type: boolean - required: true + required: false default: false jobs: - lint_all: - name: Lint All - if: ${{ github.event.inputs.LINTS == 'true' }} - uses: ./.github/workflows/reusable_lint.yml + + checks: + name: Run All Checks + if: ${{ github.event.inputs.CHECKS == 'true' }} + uses: ./.github/workflows/reusable_checks.yml + with: + SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} + secrets: inherit + + deploy-docs: + # Never deploy docs if checks haven't passed + needs: [checks] + name: Deploy Docs + if: ${{ github.event.inputs.DEPLOY_DOCS == 'true' }} + uses: ./.github/workflows/reusable_deploy_docs.yml + with: + PY_DOCS_VERSION_NAME: "test" + UPDATE_LATEST: false + secrets: inherit + + benches: + name: Benchmarks + if: ${{ github.event.inputs.BENCHES == 'true' }} + uses: ./.github/workflows/reusable_bench.yml + secrets: inherit + + min-test-wheel: + name: 'Minimum Test Wheel' + # The upload-web job uses the min-test-wheel to get the RRD + if: ${{ (github.event.inputs.MIN_TEST_WHEEL == 'true') || ((github.event.inputs.UPLOAD_GCLOUD == 'true') && ( github.event.inputs.BUILD_WEB == 'true') ) }} + uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} - secrets: - CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PLATFORM: linux + MATURIN_FEATURE_FLAGS: '--no-default-features --features extension-module' + WHEEL_ARTIFACT_NAME: '' + RRD_ARTIFACT_NAME: linux-rrd-fast + secrets: inherit - build_linux: - name: Build/Test Linux - if: ${{ github.event.inputs.BUILD_LINUX == 'true' }} - uses: ./.github/workflows/reusable_build_and_test.yml + build-linux: + name: 'Linux: Build/Test Wheels' + # The upload-wheels jobs all use the linux build to get the RRD + # TODO(jleibs): Debug why multi-line if statements don't work here + if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'linux') || ((github.event.inputs.UPLOAD_GCLOUD == 'true') && ( github.event.inputs.WHEEL_PLATFORMS != '') ) }} + + uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: linux - secrets: - CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WHEEL_ARTIFACT_NAME: linux-wheel + RRD_ARTIFACT_NAME: linux-rrd + RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }} + secrets: inherit - build_windows: - name: Build/Test Windows - if: ${{ github.event.inputs.BUILD_WINDOWS == 'true'}} - uses: ./.github/workflows/reusable_build_and_test.yml + build-windows: + name: 'Windows: Build/Test Wheels' + if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'windows') }} + uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: windows - secrets: - CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WHEEL_ARTIFACT_NAME: windows-wheel + RRD_ARTIFACT_NAME: '' + RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }} + secrets: inherit - build_macos_arm: - name: Build/Test Mac (Arm) - if: ${{ github.event.inputs.BUILD_MACOS_ARM == 'true' }} - uses: ./.github/workflows/reusable_build_and_test.yml + build-macos-arm: + name: 'Macos-Arm: Build/Test Wheels' + if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-arm') }} + uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: macos-arm - secrets: - CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WHEEL_ARTIFACT_NAME: macos-arm-wheel + RRD_ARTIFACT_NAME: '' + RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }} + secrets: inherit - build_macos_intel: - name: Build/Test Mac (Intel) - if: ${{ github.event.inputs.BUILD_MACOS_INTEL == 'true' }} - uses: ./.github/workflows/reusable_build_and_test.yml + build-macos-intel: + name: 'Macos-Intel: Build/Test Wheels' + if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-intel') }} + uses: ./.github/workflows/reusable_build_and_test_wheels.yml with: SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }} PLATFORM: macos-intel - secrets: - CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - package_and_uplaod: - name: Package and Upload - needs: [build_linux, build_windows] - if: ${{ github.event.inputs.PACKAGE == 'true' }} - uses: ./.github/workflows/reusable_package_and_upload.yml - secrets: - CALLER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WHEEL_ARTIFACT_NAME: 'macos-intel-wheel' + RRD_ARTIFACT_NAME: '' + RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }} + + secrets: inherit + + upload-wheels-linux: + name: 'Linux: Upload Wheels' + needs: [build-linux] + if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'linux') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} + uses: ./.github/workflows/reusable_upload_wheels.yml + with: + WHEEL_ARTIFACT_NAME: linux-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + upload-wheels-windows: + name: 'Windows: Upload Wheels' + needs: [build-linux, build-windows] + if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'windows') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} + uses: ./.github/workflows/reusable_upload_wheels.yml + with: + WHEEL_ARTIFACT_NAME: windows-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + upload-wheels-macos-arm: + name: 'Macos-Arm: Upload Wheels' + needs: [build-linux, build-macos-arm] + if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-arm') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} + uses: ./.github/workflows/reusable_upload_wheels.yml + with: + WHEEL_ARTIFACT_NAME: macos-arm-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + upload-wheels-macos-intel: + name: 'Macos-Intel: Upload Wheels' + needs: [build-linux, build-macos-intel] + if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-intel') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} + uses: ./.github/workflows/reusable_upload_wheels.yml + with: + WHEEL_ARTIFACT_NAME: macos-intel-wheel + RRD_ARTIFACT_NAME: linux-rrd + secrets: inherit + + build-web: + name: 'Build Web' + if: ${{ github.event.inputs.BUILD_WEB == 'true'}} + uses: ./.github/workflows/reusable_build_web.yml + with: + RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }} + secrets: inherit + + upload-web: + name: 'Upload Web' + needs: [min-test-wheel, build-web] + if: ${{ (github.event.inputs.BUILD_WEB == 'true') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }} + uses: ./.github/workflows/reusable_upload_web.yml + with: + MARK_PRERELEASE_FOR_MAINLINE: false + MARK_TAGGED_VERSION: false + RRD_ARTIFACT_NAME: linux-rrd-fast + secrets: inherit + + check-for-pr: + runs-on: ubuntu-latest + if: ${{ github.event.inputs.UPDATE_PR_SUMMARY == 'true' }} + outputs: + PR_NUMBER: ${{ steps.get_pr.outputs.PR_NUMBER }} + steps: + - name: Check if commit belongs to a PR + id: get_pr + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + pr_number=$(curl --silent --header "Authorization: Bearer ${GITHUB_TOKEN}" \ + --url "https://api.github.com/repos/${GITHUB_REPOSITORY}/commits/${GITHUB_SHA}/pulls" \ + | jq '.[] | .number') + + if [ -z "$pr_number" ]; then + echo "No PR associated with this commit" + exit 1 + else + echo "Commit is associated with PR: $pr_number" + echo "PR_NUMBER=$pr_number" >> "$GITHUB_OUTPUT" + fi + + update-pr-summary: + name: 'Update PR Summary' + # TODO(jleibs): There's no good way to have an optional needs. + # If we leave this here the job fails if we don't build all the wheels + web. + # Since this just just for testing, leave this out. We can manually run it to update + # a PR if we want to see changes. + #needs: [check-for-pr, upload-web, upload-wheels-linux, upload-wheels-windows, upload-wheels-macos-arm, upload-wheels-macos-intel] + needs: [check-for-pr] + if: ${{ github.event.inputs.UPDATE_PR_SUMMARY == 'true' }} + uses: ./.github/workflows/reusable_pr_summary.yml + with: + PR_NUMBER: ${{ needs.check-for-pr.outputs.PR_NUMBER }} + secrets: inherit diff --git a/.github/workflows/reusable_bench.yml b/.github/workflows/reusable_bench.yml new file mode 100644 index 000000000000..7a48dda8c141 --- /dev/null +++ b/.github/workflows/reusable_bench.yml @@ -0,0 +1,86 @@ +name: Reusable Bench + +on: + workflow_call: + inputs: + SAVE_BENCHES: + required: false + type: boolean + default: false + +env: + PYTHON_VERSION: "3.8" + # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses + # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html + # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html + RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings + + # See https://github.com/ericseppanen/cargo-cranky/issues/8 + RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs + + # See: https://github.com/marketplace/actions/sccache-action + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + +jobs: + +# --------------------------------------------------------------------------- + + rs-benchmarks: + name: Rust Criterion benchmarks + + runs-on: ubuntu-latest-16-cores + container: + image: rerunio/ci_docker:0.6 + steps: + - uses: actions/checkout@v3 + + - name: Set up cargo cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "build-linux" + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + # Don't update the cache -- it will be updated by the lint job + # TODO(jleibs): this job will likely run before rust.yml updates + # the cache. Better cross-job sequencing would be nice here + save-if: false + + # Sccache will cache everything else + # See: https://github.com/marketplace/actions/sccache-action + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + - name: Run benchmark + # Use bash shell so we get pipefail behavior with tee + shell: bash + run: | + cargo bench \ + --all-features \ + -p re_arrow_store \ + -p re_data_store \ + -p re_log_encoding \ + -p re_query \ + -p re_tuid \ + -- --output-format=bencher | tee output.txt + + - name: Store benchmark result + # https://github.com/benchmark-action/github-action-benchmark + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Rust Benchmark + tool: "cargo" + output-file-path: output.txt + github-token: ${{ secrets.GITHUB_TOKEN }} + + # Show alert with commit comment on detecting possible performance regression + comment-on-alert: true + alert-threshold: "125%" + fail-on-alert: true + comment-always: false # Generates too much GitHub notification spam + + # Save, results and push to GitHub only on main + save-data-file: ${{ inputs.SAVE_BENCHES }} + auto-push: ${{ inputs.SAVE_BENCHES }} + gh-pages-branch: gh-pages + benchmark-data-dir-path: dev/bench + max-items-in-chart: 30 diff --git a/.github/workflows/reusable_build_and_test_wheels.yml b/.github/workflows/reusable_build_and_test_wheels.yml new file mode 100644 index 000000000000..61dd0f5d84d4 --- /dev/null +++ b/.github/workflows/reusable_build_and_test_wheels.yml @@ -0,0 +1,293 @@ +name: Reusable Build and Test Wheels + +on: + workflow_call: + inputs: + MATURIN_FEATURE_FLAGS: + required: false + type: string + default: '--no-default-features --features pypi' + PLATFORM: + required: true + type: string + RELEASE_VERSION: + required: false + type: string + default: 'prerelease' + RRD_ARTIFACT_NAME: + required: false + type: string + default: '' + SAVE_CACHE: + required: false + type: boolean + default: false + WHEEL_ARTIFACT_NAME: + required: false + type: string + default: '' + +env: + PYTHON_VERSION: "3.8" + # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses + # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html + # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html + + # TODO(jleibs) --deny warnings causes installation of wasm-bindgen to fail on mac + # RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings + RUSTFLAGS: --cfg=web_sys_unstable_apis + + # See https://github.com/ericseppanen/cargo-cranky/issues/8 + RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs + + # See: https://github.com/marketplace/actions/sccache-action + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + +jobs: + +# --------------------------------------------------------------------------- + + set-config: + name: Set Config + runs-on: ubuntu-latest + outputs: + RUNNER: ${{ steps.set-config.outputs.runner }} + TARGET: ${{ steps.set-config.outputs.target }} + RUN_TESTS: ${{ steps.set-config.outputs.run_tests }} + CONTAINER: ${{ steps.set-config.outputs.container }} + steps: + - name: Set runner and target based on platform + id: set-config + run: | + case "${{ inputs.platform }}" in + linux) + runner="ubuntu-latest" + target="x86_64-unknown-linux-gnu" + run_tests="true" + container="{'image': 'rerunio/ci_docker:0.6'}" + ;; + windows) + runner="windows-latest" + target="x86_64-pc-windows-msvc" + run_tests="true" + container="null" + ;; + macos-arm) + runner="macos-latest" + target="aarch64-apple-darwin" + run_tests="false" + container="null" + ;; + macos-intel) + runner="macos-latest" + target="x86_64-apple-darwin" + run_tests="false" + container="null" + ;; + *) echo "Invalid platform" && exit 1 ;; + esac + echo "runner=$runner" >> "$GITHUB_OUTPUT" + echo "target=$target" >> "$GITHUB_OUTPUT" + echo "run_tests=$run_tests" >> "$GITHUB_OUTPUT" + echo "container=$container" >> "$GITHUB_OUTPUT" + +# --------------------------------------------------------------------------- + + build-wheels: + name: Build Wheels + + needs: [set-config] + + runs-on: ${{ needs.set-config.outputs.RUNNER }} + container: ${{ fromJson(needs.set-config.outputs.CONTAINER) }} + + steps: + - name: Show context + run: | + echo "GITHUB_CONTEXT": $GITHUB_CONTEXT + echo "JOB_CONTEXT": $JOB_CONTEXT + echo "INPUTS_CONTEXT": $INPUTS_CONTEXT + echo "ENV_CONTEXT": $ENV_CONTEXT + env: + ENV_CONTEXT: ${{ toJson(env) }} + GITHUB_CONTEXT: ${{ toJson(github) }} + JOB_CONTEXT: ${{ toJson(job) }} + INPUTS_CONTEXT: ${{ toJson(inputs) }} + + - uses: actions/checkout@v3 + + # Rust-cache will cache our dependencies, which is a large chunk of the build + # See: https://github.com/Swatinem/rust-cache + - name: Set up cargo cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "build-${{ inputs.PLATFORM }}" + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + # Don't update the cache -- it will be updated by the lint job + # TODO(jleibs): this job will likely run before rust.yml updates + # the cache. Better cross-job sequencing would be nice here + save-if: ${{ inputs.SAVE_CACHE }} + + # Sccache will cache everything else + # See: https://github.com/marketplace/actions/sccache-action + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + # The pip-cache setup logic doesn't work in the ubuntu docker container + # That's probably fine since we bake these deps into the container already + - name: Setup python + if: ${{ inputs.PLATFORM != 'linux' }} + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: "pip" + cache-dependency-path: "rerun_py/requirements-build.txt" + + # These should already be in the docker container, but run for good measure. A no-op install + # should be fast, and this way things don't break if we add new packages without rebuilding + # docker + - run: pip install -r rerun_py/requirements-build.txt + + # We have a nice script for that: ./scripts/setup_web.sh + # Unfortunately, we can't run that on Windows, because Windows doesn't come with + # a package manager like grown-up OSes do (at least not the CI version of Windows). + # Also we can't run it on linux because the 20.04 Docker container will install + # an old version of binaryen/wasm-opt that barfs on the `--fast-math` flag + # So we only run the script on macos, and then on Windows we do the parts of the script manually. + # On ubuntu, the correct packages are pre-installed in our docker container. + + - name: Install prerequisites for building the web-viewer Wasm (non-Windows) + if: (inputs.PLATFORM == 'macos-intel') || (inputs.PLATFORM == 'macos-arm') + shell: bash + run: ./scripts/setup_web.sh + + # The first steps of setup_web.sh, for Windows: + - name: Install wasm32 and wasm-bindgen-cli for building the web-viewer Wasm on windows + if: inputs.platform == 'windows' + shell: bash + run: rustup target add wasm32-unknown-unknown && cargo install wasm-bindgen-cli --version 0.2.84 + + # The last step of setup_web.sh, for Windows. + # Since 'winget' is not available within the GitHub runner, we download the package directly: + # See: https://github.com/marketplace/actions/engineerd-configurator + - name: Install binaryen for building the web-viewer Wasm on windows + if: inputs.platform == 'windows' + uses: engineerd/configurator@v0.0.9 + with: + name: "wasm-opt.exe" + url: "https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-windows.tar.gz" + pathInArchive: "binaryen-version_111/bin/wasm-opt.exe" + + - name: Patch Cargo.toml for pre-release + if: ${{ inputs.RELEASE_VERSION == 'prerelease' }} + # After patching the pre-release version, run cargo update. + # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing + run: | + python3 scripts/version_util.py --patch_prerelease + cargo update -w + + - name: Store the expected version + # Find the current cargo version and store it in the output: `expected_version` + shell: bash + id: expected_version + run: | + echo "EXPECTED_VERSION=$(python3 scripts/version_util.py --bare_cargo_version)" >> "$GITHUB_OUTPUT" + + - name: Check the expected version + if: ${{ inputs.RELEASE_VERSION != 'prerelease' }} + run: | + if [ "${{steps.expected_version.outputs.EXPECTED_VERSION}}" != "${{inputs.RELEASE_VERSION}}" ]; then + echo "Error: EXPECTED_VERSION (${{steps.expected_version.outputs.EXPECTED_VERSION}}) does not match RELEASE_VERSION (${{inputs.RELEASE_VERSION}})" + exit 1 + fi + + - name: Build Wheel + uses: PyO3/maturin-action@v1 + with: + maturin-version: "0.14.10" + manylinux: manylinux_2_31 + container: off + command: build + sccache: 'true' + args: | + --manifest-path rerun_py/Cargo.toml + --release + --target ${{ needs.set-config.outputs.TARGET }} + ${{ inputs.MATURIN_FEATURE_FLAGS }} + --out dist + + - name: Save wheel artifact + if: ${{ inputs.WHEEL_ARTIFACT_NAME != '' }} + uses: actions/upload-artifact@v3 + with: + name: ${{inputs.WHEEL_ARTIFACT_NAME}} + path: dist + + # --------------------------------------------------------------------------- + # Test the wheel + + - name: Install wheel dependencies + if: needs.set-config.outputs.RUN_TESTS == 'true' + # First we install the dependencies manually so we can use `--no-index` when installing the wheel. + # This needs to be a separate step for some reason or the following step fails + # TODO(jleibs): pull these deps from pyproject.toml + # TODO(jleibs): understand why deps can't be installed in the same step as the wheel + shell: bash + run: | + pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2 + + - name: Install built wheel + if: needs.set-config.outputs.RUN_TESTS == 'true' + # Now install the wheel using a specific version and --no-index to guarantee we get the version from + # the pre-dist folder. Note we don't use --force-reinstall here because --no-index means it wouldn't + # find the dependencies to reinstall them. + shell: bash + run: | + pip uninstall rerun-sdk + pip install rerun-sdk==${{ steps.expected_version.outputs.EXPECTED_VERSION }} --no-index --find-links dist + + - name: Verify built wheel version + if: needs.set-config.outputs.RUN_TESTS == 'true' + shell: bash + run: | + python3 -m rerun --version + which rerun + rerun --version + + - name: Run unit tests + if: needs.set-config.outputs.RUN_TESTS == 'true' + shell: bash + run: cd rerun_py/tests && pytest + + - name: Run e2e test + if: needs.set-config.outputs.RUN_TESTS == 'true' + shell: bash + run: RUST_LOG=debug scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed + + - name: Cache RRD dataset + if: needs.set-config.outputs.RUN_TESTS == 'true' + id: dataset + uses: actions/cache@v3 + with: + path: examples/python/colmap/dataset/ + # TODO(jleibs): Derive this key from the invocation below + key: colmap-dataset-colmap-fiat-v0 + + - name: Generate Embedded RRD file + if: needs.set-config.outputs.RUN_TESTS == 'true' + shell: bash + # If you change the line below you should almost definitely change the `key:` line above by giving it a new, unique name + run: | + mkdir rrd + pip install -r examples/python/colmap/requirements.txt + python3 examples/python/colmap/main.py --dataset colmap_fiat --resize 800x600 --save rrd/colmap_fiat.rrd + + # All platforms are currently creating the same rrd file, upload one of them + - name: Save RRD artifact + if: ${{ (needs.set-config.outputs.RUN_TESTS == 'true') && (inputs.RRD_ARTIFACT_NAME != '') }} + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.RRD_ARTIFACT_NAME }} + path: rrd + diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml new file mode 100644 index 000000000000..a3dd17bbaf1c --- /dev/null +++ b/.github/workflows/reusable_build_web.yml @@ -0,0 +1,83 @@ +name: Reusable Build Web + +on: + workflow_call: + inputs: + RELEASE_VERSION: + required: false + type: string + default: 'prerelease' + +env: + # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses + # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html + # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html + RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings + + # See https://github.com/ericseppanen/cargo-cranky/issues/8 + RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs + +jobs: + + rs-build-web-viewer: + name: Build Web (wasm32 + wasm-bindgen) + permissions: + contents: "read" + id-token: "write" + + runs-on: ubuntu-latest-16-cores + + container: + image: rerunio/ci_docker:0.6 + + steps: + - uses: actions/checkout@v3 + + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: 1.67.0 + target: wasm32-unknown-unknown + override: true + + - name: Set up cargo cache + uses: Swatinem/rust-cache@v2 + with: + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + # See: https://github.com/rerun-io/rerun/pull/497 + save-if: false + + - name: Patch Cargo.toml for pre-release + if: ${{ inputs.RELEASE_VERSION == 'prerelease' }} + # After patching the pre-release version, run cargo update. + # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing + run: | + python3 scripts/version_util.py --patch_prerelease + cargo update -w + + - name: Store the expected version + # Find the current cargo version and store it in the output: `expected_version` + shell: bash + id: expected_version + run: | + echo "EXPECTED_VERSION=$(python3 scripts/version_util.py --bare_cargo_version)" >> "$GITHUB_OUTPUT" + + - name: Check the expected version + if: ${{ inputs.RELEASE_VERSION != 'prerelease' }} + run: | + if [ "${{steps.expected_version.outputs.EXPECTED_VERSION}}" != "${{inputs.RELEASE_VERSION}}" ]; then + echo "Error: EXPECTED_VERSION (${{steps.expected_version.outputs.EXPECTED_VERSION}}) does not match RELEASE_VERSION (${{inputs.RELEASE_VERSION}})" + exit 1 + fi + + - name: Build web-viewer (release) + uses: actions-rs/cargo@v1 + with: + command: run + args: --locked -p re_build_web_viewer -- --release + + - name: Upload web assets + uses: actions/upload-artifact@v3 + with: + name: web_viewer + path: web_viewer diff --git a/.github/workflows/reusable_checks.yml b/.github/workflows/reusable_checks.yml new file mode 100644 index 000000000000..e62e7d17419f --- /dev/null +++ b/.github/workflows/reusable_checks.yml @@ -0,0 +1,321 @@ +name: 'Checks: Lints, Tests, Docs' + +on: + workflow_call: + inputs: + SAVE_CACHE: + required: false + type: boolean + default: false + SAVE_PY_DOCS: + required: false + type: boolean + default: false + SAVE_PY_DOCS_AS: + required: false + type: string + default: "" + SAVE_RUST_DOCS: + required: false + type: boolean + default: false + +env: + PYTHON_VERSION: "3.8" + # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses + # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html + # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html + RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings + + # See https://github.com/ericseppanen/cargo-cranky/issues/8 + RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs + + # See: https://github.com/marketplace/actions/sccache-action + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + +jobs: + +# --------------------------------------------------------------------------- + + py-lints: + name: Python lints (black, mypy, flake8) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + just-version: 1.5 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: "pip" + cache-dependency-path: "rerun_py/requirements-lint.txt" + + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install -r rerun_py/requirements-lint.txt + + - name: Lint Python + run: | + just py-lint + + - name: Check requirements + run: | + just py-requirements + +# --------------------------------------------------------------------------- + + py-test-docs: + name: Test Python Docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.8" + cache: "pip" + cache-dependency-path: "rerun_py/requirements-doc.txt" + + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install -r rerun_py/requirements-doc.txt + + - name: Build via mkdocs + run: | + mkdocs build -f rerun_py/mkdocs.yml + +# --------------------------------------------------------------------------- + + rs-lints: + name: Rust lints (fmt, check, cranky, tests, doc) + runs-on: ubuntu-latest-16-cores + container: + image: rerunio/ci_docker:0.6 + steps: + - name: Show context + run: | + echo "GITHUB_CONTEXT": $GITHUB_CONTEXT + echo "JOB_CONTEXT": $JOB_CONTEXT + echo "INPUTS_CONTEXT": $INPUTS_CONTEXT + echo "ENV_CONTEXT": $ENV_CONTEXT + env: + ENV_CONTEXT: ${{ toJson(env) }} + GITHUB_CONTEXT: ${{ toJson(github) }} + JOB_CONTEXT: ${{ toJson(job) }} + INPUTS_CONTEXT: ${{ toJson(inputs) }} + + - uses: actions/checkout@v3 + + # Rust-cache will cache our dependencies, which is a large chunk of the build + # See: https://github.com/Swatinem/rust-cache + - name: Set up cargo cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "build-linux" + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + # Don't update the cache -- it will be updated by the lint job + # TODO(jleibs): this job will likely run before rust.yml updates + # the cache. Better cross-job sequencing would be nice here + save-if: ${{ inputs.SAVE_CACHE }} + + # Sccache will cache everything else + # See: https://github.com/marketplace/actions/sccache-action + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + + # First do our check with --locked to make sure `Cargo.lock` is up to date + - name: Check all features + uses: actions-rs/cargo@v1 + with: + command: check + args: --locked --all-features + + - name: Rustfmt + uses: actions-rs/cargo@v1 + with: + command: fmt + args: --all -- --check + + - name: Cranky + uses: actions-rs/cargo@v1 + with: + command: cranky + args: --all-targets --all-features -- --deny warnings + + # Check a few important permutations of the feature flags for our `rerun` library: + - name: Check rerun with `--no-default-features`` + uses: actions-rs/cargo@v1 + with: + command: cranky + args: --locked -p rerun --no-default-features + + - name: Check rerun with `--features sdk` + uses: actions-rs/cargo@v1 + with: + command: cranky + args: --locked -p rerun --no-default-features --features sdk + + - name: Test doc-tests + uses: actions-rs/cargo@v1 + with: + command: test + args: --doc --all-features + + - name: cargo doc + uses: actions-rs/cargo@v1 + with: + command: doc + args: --no-deps --all-features + + - name: cargo doc --document-private-items + uses: actions-rs/cargo@v1 + with: + command: doc + args: --document-private-items --no-deps --all-features + + - name: Test + uses: actions-rs/cargo@v1 + with: + command: test + args: --all-targets --all-features + +# --------------------------------------------------------------------------- + + rs-check-wasm: + name: Check Rust web build (wasm32 + wasm-bindgen) + runs-on: ubuntu-latest-16-cores + container: + image: rerunio/ci_docker:0.6 + steps: + - uses: actions/checkout@v3 + + - uses: actions-rs/toolchain@v1 + with: + profile: default + toolchain: 1.67.1 + target: wasm32-unknown-unknown + override: true + + - name: Set up cargo cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "build-wasm" + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + # See: https://github.com/rerun-io/rerun/pull/497 + save-if: ${{ inputs.SAVE_CACHE }} + + # See: https://github.com/marketplace/actions/sccache-action + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + - name: clippy check re_viewer wasm32 + run: ./scripts/clippy_wasm.sh + + - name: Check re_renderer examples wasm32 + uses: actions-rs/cargo@v1 + with: + command: check + args: --locked --target wasm32-unknown-unknown --target-dir target_wasm -p re_renderer --examples + + - name: Build web-viewer (debug) + uses: actions-rs/cargo@v1 + with: + command: run + args: --locked -p re_build_web_viewer -- --debug + +# --------------------------------------------------------------------------- + + toml-lints: + name: Lint TOML files + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v3 + + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: 1.67.1 + override: true + + - name: Set up cargo cache + uses: Swatinem/rust-cache@v2 + with: + # See: https://github.com/rerun-io/rerun/pull/497 + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + # Don't update the cache -- it will be updated by the lint job + # TODO(jleibs): this job will likely run before rust.yml updates + # the cache. Better cross-job sequencing would be nice here + save-if: false + + - name: Install taplo-cli + uses: baptiste0928/cargo-install@v1 + with: + crate: taplo-cli + + - name: Taplo check + run: | + taplo fmt --check + +# --------------------------------------------------------------------------- + + misc-rerun-lints: + name: Rerun lints + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.8" + + - name: Rerun lints + run: | + ./scripts/lint.py + + - name: Check for too large files + run: | + ./scripts/check_large_files.sh + +# --------------------------------------------------------------------------- + + spell-check: + name: Spell Check + runs-on: ubuntu-latest + steps: + - name: Checkout Actions Repository + uses: actions/checkout@v3 + + - name: Check spelling of entire workspace + uses: crate-ci/typos@master + +# --------------------------------------------------------------------------- + + rs-cargo-deny: + name: Cargo Deny ${{ matrix.platform }} + runs-on: ubuntu-latest + container: + image: rerunio/ci_docker:0.6 + + steps: + - uses: actions/checkout@v3 + + # See: https://github.com/marketplace/actions/sccache-action + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + - name: Cargo Deny + shell: bash + id: expected_version + run: ./scripts/cargo_deny.sh diff --git a/.github/workflows/reusable_deploy_docs.yml b/.github/workflows/reusable_deploy_docs.yml new file mode 100644 index 000000000000..f2bceec77ce3 --- /dev/null +++ b/.github/workflows/reusable_deploy_docs.yml @@ -0,0 +1,151 @@ +name: 'Reusable Deploy Docs' + +on: + workflow_call: + inputs: + PY_DOCS_VERSION_NAME: + required: true + type: string + UPDATE_LATEST: + required: false + type: boolean + default: false + +env: + PYTHON_VERSION: "3.8" + # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses + # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html + # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html + RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings + + # See https://github.com/ericseppanen/cargo-cranky/issues/8 + RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs + + # See: https://github.com/marketplace/actions/sccache-action + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + +jobs: + +# --------------------------------------------------------------------------- + + py-deploy-docs: + name: Python + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Don't do a shallow clone + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.8" + cache: "pip" + cache-dependency-path: "rerun_py/requirements-doc.txt" + + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install -r rerun_py/requirements-doc.txt + + - name: Set up git author + run: | + remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git config --global user.name "${GITHUB_ACTOR}" + git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Mike will incrementally update the existing gh-pages branch + # We then check it out, and reset it to a new orphaned branch, which we force-push to origin + # to make sure we don't accumulate unnecessary history in gh-pages branch + - name: Deploy via mike # https://github.com/jimporter/mike + if: ${{ inputs.UPDATE_LATEST }} + run: | + git fetch + mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python -u ${{inputs.PY_DOCS_VERSION_NAME}} latest + git checkout gh-pages + git checkout --orphan gh-pages-orphan + git commit -m "Update docs for ${GITHUB_SHA}" + git push origin gh-pages-orphan:gh-pages -f + + # Mike will incrementally update the existing gh-pages branch + # We then check it out, and reset it to a new orphaned branch, which we force-push to origin + # to make sure we don't accumulate unnecessary history in gh-pages branch + - name: Deploy tag via mike # https://github.com/jimporter/mike + if: ${{ ! inputs.UPDATE_LATEST }} + run: | + git fetch + mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python ${{inputs.PY_DOCS_VERSION_NAME}} + git checkout gh-pages + git checkout --orphan gh-pages-orphan + git commit -m "Update docs for ${GITHUB_SHA}" + git push origin gh-pages-orphan:gh-pages -f + + +# --------------------------------------------------------------------------- + + rs-deploy-docs: + name: Rust + runs-on: ubuntu-latest-16-cores + container: + image: rerunio/ci_docker:0.6 + steps: + - name: Show context + run: | + echo "GITHUB_CONTEXT": $GITHUB_CONTEXT + echo "JOB_CONTEXT": $JOB_CONTEXT + echo "INPUTS_CONTEXT": $INPUTS_CONTEXT + echo "ENV_CONTEXT": $ENV_CONTEXT + env: + ENV_CONTEXT: ${{ toJson(env) }} + GITHUB_CONTEXT: ${{ toJson(github) }} + JOB_CONTEXT: ${{ toJson(job) }} + INPUTS_CONTEXT: ${{ toJson(inputs) }} + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Don't do a shallow clone since we need to push gh-pages + + # Rust-cache will cache our dependencies, which is a large chunk of the build + # See: https://github.com/Swatinem/rust-cache + - name: Set up cargo cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "build-linux" + env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY + save-if: false + + # Sccache will cache everything else + # See: https://github.com/marketplace/actions/sccache-action + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + - name: cargo doc --document-private-items + uses: actions-rs/cargo@v1 + with: + command: doc + args: --document-private-items --no-deps --all-features + + - name: Set up git author + run: | + remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git config --global user.name "${GITHUB_ACTOR}" + git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up ghp-import + run: pip install ghp-import + + - name: Patch in a redirect page + run: echo "" > target/doc/index.html + env: + REDIRECT_CRATE: rerun + + # See: https://github.com/c-w/ghp-import + - name: Deploy the docs + run: | + git fetch + python3 -m ghp_import -n -p -x docs/rust/head target/doc/ -m "Update the rust docs" diff --git a/.github/workflows/reusable_pr_summary.yml b/.github/workflows/reusable_pr_summary.yml new file mode 100644 index 000000000000..c148bcf076f9 --- /dev/null +++ b/.github/workflows/reusable_pr_summary.yml @@ -0,0 +1,52 @@ +name: Reusable PR Summary + +on: + workflow_call: + inputs: + PR_NUMBER: + required: true + type: string + +jobs: + pr-summary: + name: Create HTML summary for PR + + permissions: + contents: "read" + id-token: "write" + + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - id: "auth" + uses: google-github-actions/auth@v1 + with: + workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' + with: + version: '>= 363.0.0' + + - name: Install deps + run: pip install google-cloud-storage Jinja2 PyGithub # NOLINT + + - name: Render HTML template + run: | + python scripts/generate_pr_summary.py \ + --github-token ${{secrets.GITHUB_TOKEN}} \ + --github-repository ${GITHUB_REPOSITORY} \ + --pr-number ${{ inputs.PR_NUMBER }} \ + --upload + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ inputs.PR_NUMBER }} diff --git a/.github/workflows/reusable_upload_web.yml b/.github/workflows/reusable_upload_web.yml new file mode 100644 index 000000000000..16142cfc654b --- /dev/null +++ b/.github/workflows/reusable_upload_web.yml @@ -0,0 +1,116 @@ +name: Reusable Upload Web + +on: + workflow_call: + inputs: + MARK_PRERELEASE_FOR_MAINLINE: + required: false + type: boolean + default: false + MARK_TAGGED_VERSION: + required: false + type: boolean + default: false + RELEASE_VERSION: + required: false + type: string + default: 'prerelease' + RRD_ARTIFACT_NAME: + required: false + type: string + default: '' + # We need this because PRs use a merged commit but we really want + # to track uploads based on the source commit. + UPLOAD_COMMIT_OVERRIDE: + required: false + type: string + default: '' + +jobs: + + upload-web: + name: Upload web build to google cloud (wasm32 + wasm-bindgen) + permissions: + contents: "read" + id-token: "write" + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Download RRD + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.RRD_ARTIFACT_NAME }} + path: rrd + + - name: Download Web Viewer + uses: actions/download-artifact@v3 + with: + name: web_viewer + path: web_viewer + + + # Upload the wasm, html etc to a Google cloud bucket: + - id: "auth" + uses: google-github-actions/auth@v1 + with: + workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + + - name: Add SHORT_SHA env property with commit short sha + run: | + if [ -z "${{ inputs.UPLOAD_COMMIT_OVERRIDE }}" ]; then + USED_SHA=${{ github.sha }} + else + USED_SHA=${{ inputs.UPLOAD_COMMIT_OVERRIDE }} + fi + echo "SHORT_SHA=$(echo $USED_SHA | cut -c1-7)" >> $GITHUB_ENV + + - name: "Upload web-viewer (commit)" + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "web_viewer" + destination: "rerun-web-viewer/commit/${{env.SHORT_SHA}}" + parent: false + + - name: "Upload RRD (commit)" + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "rrd" + destination: "rerun-example-rrd/commit/${{env.SHORT_SHA}}" + parent: false + + - name: "Upload web-viewer (prerelease)" + if: inputs.MARK_PRERELEASE_FOR_MAINLINE + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "web_viewer" + destination: "rerun-web-viewer/prerelease" + parent: false + + + - name: "Upload RRD (prerelease)" + if: inputs.MARK_PRERELEASE_FOR_MAINLINE + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "rrd" + destination: "rerun-example-rrd/prerelease" + parent: false + + - name: "Upload web-viewer (tagged)" + if: inputs.MARK_TAGGED_VERSION + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "web_viewer" + destination: "rerun-web-viewer/version/${{inputs.RELEASE_VERSION}}" + parent: false + + - name: "Upload RRD (tagged)" + if: inputs.MARK_TAGGED_VERSION + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "rrd" + destination: "rerun-example-rrd/version/${{inputs.RELEASE_VERSION}}" + parent: false diff --git a/.github/workflows/reusable_upload_wheels.yml b/.github/workflows/reusable_upload_wheels.yml new file mode 100644 index 000000000000..575b2a3cffb1 --- /dev/null +++ b/.github/workflows/reusable_upload_wheels.yml @@ -0,0 +1,96 @@ +name: Reusable Upload Wheels + +on: + workflow_call: + inputs: + RRD_ARTIFACT_NAME: + required: false + type: string + default: '' + WHEEL_ARTIFACT_NAME: + required: false + type: string + default: '' + +jobs: + + upload-wheel: + name: Upload Wheel to google cloud + + runs-on: ubuntu-latest + + container: + image: rerunio/ci_docker:0.6 + + permissions: + contents: "read" + id-token: "write" + + steps: + - name: Show context + run: | + echo "GITHUB_CONTEXT": $GITHUB_CONTEXT + echo "JOB_CONTEXT": $JOB_CONTEXT + echo "INPUTS_CONTEXT": $INPUTS_CONTEXT + echo "ENV_CONTEXT": $ENV_CONTEXT + echo "MATRIX_CONTEXT": $MATRIX_CONTEXT + env: + ENV_CONTEXT: ${{ toJson(env) }} + GITHUB_CONTEXT: ${{ toJson(github) }} + JOB_CONTEXT: ${{ toJson(job) }} + INPUTS_CONTEXT: ${{ toJson(inputs) }} + + - uses: actions/checkout@v3 + + - name: Download RRD + uses: actions/download-artifact@v3 + with: + name: ${{inputs.RRD_ARTIFACT_NAME}} + path: rrd + + - name: Download Wheel + uses: actions/download-artifact@v3 + with: + name: ${{inputs.WHEEL_ARTIFACT_NAME}} + path: pre-dist + + - name: Unpack the wheel + shell: bash + run: | + mkdir unpack-dist + wheel unpack pre-dist/*.whl --dest unpack-dist + + - name: Get the folder name + shell: bash + id: get_folder_name + run: | + echo "PKG_FOLDER=$(ls unpack-dist)" >> "$GITHUB_OUTPUT" + + - name: Insert the rrd + shell: bash + # If you change the line below you should almost definitely change the `key:` line + # in 'Cache RRD dataset'reusable_build_and_test.yml + run: | + cp rrd/colmap_fiat.rrd unpack-dist/${{ steps.get_folder_name.outputs.PKG_FOLDER }}/rerun_sdk/rerun_demo/colmap_fiat.rrd + + - name: Repack the wheel + shell: bash + run: | + mkdir dist + wheel pack unpack-dist/${{ steps.get_folder_name.outputs.PKG_FOLDER }} --dest dist/ + + - id: "auth" + uses: google-github-actions/auth@v1 + with: + workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + + - name: Add SHORT_SHA env property with commit short sha + run: echo "SHORT_SHA=`echo ${{github.sha}} | cut -c1-7`" >> $GITHUB_ENV + + - name: "Upload Wheel" + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: "dist" + destination: "rerun-builds/commit/${{env.SHORT_SHA}}/wheels" + parent: false diff --git a/.vscode/extensions.json b/.vscode/extensions.json index aae305c13ea6..32731cdfadeb 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -2,17 +2,18 @@ // See https://go.microsoft.com/fwlink/?LinkId=827846 // for the documentation about the extensions.json format "recommendations": [ - "charliermarsh.ruff", // Ruff for linting + "charliermarsh.ruff", + "github.vscode-github-actions", "ms-python.python", - "ms-vsliveshare.vsliveshare", // Live Share + "ms-vsliveshare.vsliveshare", "polymeilex.wgsl", - "rust-lang.rust-analyzer", // Rust-analyzer + "rust-lang.rust-analyzer", "serayuzgur.crates", "streetsidesoftware.code-spell-checker", - "tamasfe.even-better-toml", // TOML LSP - "vadimcn.vscode-lldb", // CodeLLDB - "wayou.vscode-todo-highlight", // TODO Highlight - "webfreak.debug", // Native Debug - "zxh404.vscode-proto3", // vscode-proto3 + "tamasfe.even-better-toml", + "vadimcn.vscode-lldb", + "wayou.vscode-todo-highlight", + "webfreak.debug", + "zxh404.vscode-proto3", ] } diff --git a/crates/re_log_types/src/component_types/arrow_convert_shims.rs b/crates/re_log_types/src/component_types/arrow_convert_shims.rs index 8d196842169d..b78ff5f4c065 100644 --- a/crates/re_log_types/src/component_types/arrow_convert_shims.rs +++ b/crates/re_log_types/src/component_types/arrow_convert_shims.rs @@ -79,6 +79,7 @@ impl<'a> Iterator for BufferBinaryArrayIter<'a> { /// Internal `ArrowArray` helper to iterate over a `BinaryArray` while exposing Buffer slices pub struct BufferBinaryArray; +#[cfg(not(target_os = "windows"))] extern "C" { fn do_not_call_into_iter(); // we never define this function, so the linker will fail } diff --git a/crates/re_log_types/src/component_types/mod.rs b/crates/re_log_types/src/component_types/mod.rs index 61066b8c8f58..a65650364660 100644 --- a/crates/re_log_types/src/component_types/mod.rs +++ b/crates/re_log_types/src/component_types/mod.rs @@ -210,6 +210,7 @@ where pub struct FastFixedSizeListArray(std::marker::PhantomData); +#[cfg(not(target_os = "windows"))] extern "C" { fn do_not_call_into_iter(); // we never define this function, so the linker will fail } diff --git a/scripts/cargo_deny.sh b/scripts/cargo_deny.sh new file mode 100755 index 000000000000..41b65fcda881 --- /dev/null +++ b/scripts/cargo_deny.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -eu +script_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) +cd "$script_path/.." +set -x + +# cargo install cargo-deny +cargo deny --all-features --log-level error --target aarch64-apple-darwin check +cargo deny --all-features --log-level error --target i686-pc-windows-gnu check +cargo deny --all-features --log-level error --target i686-pc-windows-msvc check +cargo deny --all-features --log-level error --target i686-unknown-linux-gnu check +cargo deny --all-features --log-level error --target wasm32-unknown-unknown check +cargo deny --all-features --log-level error --target x86_64-apple-darwin check +cargo deny --all-features --log-level error --target x86_64-pc-windows-gnu check +cargo deny --all-features --log-level error --target x86_64-pc-windows-msvc check +cargo deny --all-features --log-level error --target x86_64-unknown-linux-gnu check +cargo deny --all-features --log-level error --target x86_64-unknown-linux-musl check +cargo deny --all-features --log-level error --target x86_64-unknown-redox check diff --git a/scripts/generate_pr_summary.py b/scripts/generate_pr_summary.py new file mode 100644 index 000000000000..a4ff04dabd71 --- /dev/null +++ b/scripts/generate_pr_summary.py @@ -0,0 +1,89 @@ +""" +Script to generate a PR summary page. + +This script combines the GitHub and google cloud storage APIs +to find and link to the builds associated with a given PR. + +This is expected to be run by the `reusable_pr_summary.yml` GitHub workflow. + +Requires the following packages: + pip install google-cloud-storage Jinja2 PyGithub # NOLINT +""" + +import argparse +import io +import os +from typing import Any, Dict + +from github import Github # NOLINT +from google.cloud import storage +from jinja2 import Template + + +def generate_pr_summary(github_token: str, github_repository: str, pr_number: int, upload: bool) -> None: + # Initialize the GitHub and GCS clients + gh = Github(github_token) # NOLINT + gcs_client = storage.Client() + + # Get the list of commits associated with the PR + repo = gh.get_repo(github_repository) + pull = repo.get_pull(pr_number) + all_commits = [commit.sha for commit in pull.get_commits()] + all_commits.reverse() + + # Prepare the found_builds list + found_builds = [] + viewer_bucket = gcs_client.bucket("rerun-web-viewer") + wheels_bucket = gcs_client.bucket("rerun-builds") + + for commit in all_commits: + commit_short = commit[:7] + print("Checking commit: {}...".format(commit_short)) + + found: Dict[str, Any] = {} + + # Check if there is a hosted app for the current commit + commit_blob = viewer_bucket.blob(f"commit/{commit_short}/index.html") + if commit_blob.exists(): + print("Found web assets commit: {}".format(commit_short)) + found["hosted_app"] = f"https://app.rerun.io/commit/{commit_short}" + + # Get the wheel files for the commit + wheel_blobs = list(wheels_bucket.list_blobs(prefix=f"commit/{commit_short}/wheels")) + wheels = [f"https://storage.googleapis.com/{blob.bucket.name}/{blob.name}" for blob in wheel_blobs] + if wheels: + print("Found wheels for commit: {}".format(commit_short)) + found["wheels"] = wheels + + if found: + found["commit"] = commit_short + found_builds.append(found) + + template_path = os.path.join(os.path.dirname(os.path.relpath(__file__)), "templates/pr_results_summary.html") + + # Render the Jinja template with the found_builds variable + with open(template_path) as f: + template = Template(f.read()) + + buffer = io.BytesIO(template.render(found_builds=found_builds, pr_number=pr_number).encode("utf-8")) + buffer.seek(0) + + if upload: + upload_blob = wheels_bucket.blob(f"pull_request/{pr_number}/index.html") + print("Uploading results to {}".format(upload_blob)) + upload_blob.upload_from_file(buffer, content_type="text/html") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Generate a PR summary page") + parser.add_argument("--github-token", required=True, help="GitHub token") + parser.add_argument("--github-repository", required=True, help="GitHub repository") + parser.add_argument("--pr-number", required=True, type=int, help="PR number") + parser.add_argument("--upload", action="store_true", help="Upload the summary page to GCS") + args = parser.parse_args() + + generate_pr_summary(args.github_token, args.github_repository, args.pr_number, args.upload) + + +if __name__ == "__main__": + main() diff --git a/scripts/templates/pr_results_summary.html b/scripts/templates/pr_results_summary.html new file mode 100644 index 000000000000..08667bd36e79 --- /dev/null +++ b/scripts/templates/pr_results_summary.html @@ -0,0 +1,48 @@ + + + + + Build Summary + + + + +

Build Summary for #{{pr_number}}

+ + {% for build in found_builds %} +
+

Commit: {{ build.commit }}

+ {% if build.hosted_app %} +
+

Hosted App:

+ {{ build.hosted_app }} +
+ {% endif %} + {% if build.wheels %} +
+

Wheels:

+ +
+ {% endif %} +
+ {% endfor %} + + + diff --git a/scripts/version_util.py b/scripts/version_util.py index 52f555966f23..e694f24c0b80 100755 --- a/scripts/version_util.py +++ b/scripts/version_util.py @@ -7,12 +7,10 @@ --patch_prerelease: This will patch the version in rerun/Cargo.toml with the current git sha. This is intended to create a prerelease version for continuous releases. - --check_version: This will check that the version in rerun/Cargo.toml matches the version in the tag name from - `GITHUB_REF_NAME`. This is intended to be used to check that the version number in Cargo.toml is correct before - creating a release on PyPI. If the versions don't match, an exception will be raised. + --bare_cargo_version Outputs the bare cargo version. This is helpful for setting an environment variable, such as: + EXPECTED_VERSION=$(python3 scripts/version_util.py --bare_cargo_version) """ -import os import re import subprocess import sys @@ -24,9 +22,6 @@ # A regex to match the version number in Cargo.toml as SemVer, e.g., 1.2.3-alpha.0 CARGO_VERSION_REGEX: Final = r"^version\s*=\s*\"(.+)\"$" -# A regex to match the version number in the tag name, e.g. v1.2.3 -VERSION_TAG_REGEX: Final = r"^v(.+)$" - def get_cargo_version(cargo_toml: str) -> semver.VersionInfo: """Using regex, parse the version number from Cargo.toml.""" @@ -44,24 +39,6 @@ def get_git_sha() -> str: return subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).decode("utf-8").strip() -def get_ref_name_version() -> semver.VersionInfo: - """Return the parsed tag version from the GITHUB_REF_NAME environment variable.""" - - # This is the branch, or tag name that triggered the workflow. - ref_name = os.environ.get("GITHUB_REF_NAME") - - if ref_name is None: - raise Exception("GITHUB_REF_NAME environment variable not set") - - # Extract the version number from the tag name - match = re.search(VERSION_TAG_REGEX, ref_name) - - if match is None: - raise Exception("Could not find valid version number in GITHUB_REF_NAME") - - return semver.parse_version_info(match.groups()[0]) - - def patch_cargo_version(cargo_toml: str, new_version: str) -> str: """Patch the version number in Cargo.toml with `new_version`.""" @@ -99,13 +76,6 @@ def main() -> None: with open("Cargo.toml", "w") as f: f.write(new_cargo_toml) - elif sys.argv[1] == "--check_version": - ref_version = get_ref_name_version() - if cargo_version != ref_version: - raise Exception( - f"Version number in Cargo.toml ({cargo_version}) does not match tag version ({ref_version})" - ) - print(f"Version numbers match: {cargo_version} == {ref_version}") elif sys.argv[1] == "--bare_cargo_version": # Print the bare cargo version. NOTE: do not add additional formatting here. This output # is expected to be fed into an environment variable.