Replace setup scripts with pixi (#4302)

### What Part of #4170 - [x] `scripts/setup_dev.sh` - [x] ~~`cargo` tools~~ these continue to be installed via `cargo` - [x] ~~`pngcrush`~~ `pixi` does not have it, removed usage of it instead because it only seemed to have limited effects in my testing, and PIL already optimizes when saving. - [x] ~~`nox` via `pipx`~~ added `nox` directly - [x] `clang-format` - [x] `flatbuffers` - [x] `scripts/setup_web.sh` - [x] `wasm32-unknown-unknown` target (already part of `rust-toolchain` file) - [x] `binaryen` for `wasm-opt` only - [x] `scripts/setup.sh` - [x] `cmake` - Also installs a long list of _runtime_ dependencies (e.g. `gtk3`, `openssl`, `apache-arrow`), these should be specified in documentation somewhere for the most common linux distributions. - [x] `ci_docker/Dockerfile` - [x] Uses `setup_web.sh`, but it doesn't need to. We just need to ensure it installs the targets specified in `rust-toolchain` - [x] CI workflows should not use any `setup` scripts There is still a long-tail of things that need to be done before we can delete the scripts, and the scope of that is too large for this PR. ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested [demo.rerun.io](https://demo.rerun.io/pr/4302) (if applicable) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG - [PR Build Summary](https://build.rerun.io/pr/4302) - [Docs preview](https://rerun.io/preview/472fd7dc27df1b8bda0ceb02fc027c0816226d7a/docs)  - [Examples preview](https://rerun.io/preview/472fd7dc27df1b8bda0ceb02fc027c0816226d7a/examples)  - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)
rerun-io · Nov 23, 2023 · 73d570f · 73d570f
1 parent 13ac376
commit 73d570f
Show file tree

Hide file tree

Showing 20 changed files with 1,109 additions and 279 deletions.
diff --git a/.github/workflows/contrib_checks.yml b/.github/workflows/contrib_checks.yml
@@ -287,7 +287,6 @@ jobs:
       - uses: prefix-dev/setup-pixi@v0.3.0
         with:
           pixi-version: v0.6.0
-          cache: true
 
       # TODO(emilk): make this work somehow. Right now this just results in
       # > Compiler: GNU 12.3.0 (/__w/rerun/rerun/.pixi/env/bin/x86_64-conda-linux-gnu-c++)

diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml
@@ -131,7 +131,6 @@ jobs:
     with:
       CONCURRENCY: pr-${{ github.event.pull_request.number }}
       WHEEL_ARTIFACT_NAME: linux-wheel-fast
-      SOURCE_LINK_COMMIT_OVERRIDE: ${{ github.event.pull_request.head.sha }}
     secrets: inherit
 
   track-sizes:

diff --git a/.github/workflows/reusable_build_and_upload_rerun_c.yml b/.github/workflows/reusable_build_and_upload_rerun_c.yml
@@ -147,7 +147,8 @@ jobs:
         id: get-sha
         shell: bash
         run: |
-          echo "sha=$(echo ${{ inputs.RELEASE_COMMIT || github.sha }} | cut -c1-7)" >> "$GITHUB_OUTPUT"
+          full_commit="${{ inputs.RELEASE_COMMIT || ((github.event_name == 'pull_request' && github.event.pull_request.head.ref) || github.sha) }}"
+          echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT"
 
       - name: "Upload rerun_c (commit)"
         uses: google-github-actions/upload-cloud-storage@v1
@@ -163,3 +164,4 @@ jobs:
           path: "./target/${{ needs.set-config.outputs.TARGET }}/release/${{ needs.set-config.outputs.LIB_NAME }}"
           destination: "rerun-builds/adhoc/${{inputs.ADHOC_NAME}}/rerun_c/${{ inputs.PLATFORM }}"
           parent: false
+
diff --git a/.github/workflows/reusable_build_and_upload_rerun_cli.yml b/.github/workflows/reusable_build_and_upload_rerun_cli.yml
@@ -135,56 +135,20 @@ jobs:
           service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
           targets: ${{ needs.set-config.outputs.TARGET }}
 
-      # The pip-cache setup logic doesn't work in the ubuntu docker container
-      # That's probably fine since we bake these deps into the container already
-      - name: Setup python
-        if: ${{ inputs.PLATFORM != 'linux' }}
-        uses: actions/setup-python@v4
+      - uses: prefix-dev/setup-pixi@v0.3.0
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
-          cache: "pip"
-          cache-dependency-path: "scripts/ci/requirements.txt"
-
-      # These should already be in the docker container, but run for good measure. A no-op install
-      # should be fast, and this way things don't break if we add new packages without rebuilding
-      # docker
-      - run: pip install -r scripts/ci/requirements.txt
-
-      # We have a nice script for that: ./scripts/setup_web.sh
-      # Unfortunately, we can't run that on Windows, because Windows doesn't come with
-      # a package manager like grown-up OSes do (at least not the CI version of Windows).
-      # Also we can't run it on linux because the 20.04 Docker container will install
-      # an old version of binaryen/wasm-opt that barfs on the `--fast-math` flag
-      # So we only run the script on macos, and then on Windows we do the parts of the script manually.
-      # On ubuntu, the correct packages are pre-installed in our docker container.
-
-      - name: Install prerequisites for building the web-viewer Wasm (non-Windows)
-        if: (inputs.PLATFORM == 'macos-intel') || (inputs.PLATFORM == 'macos-arm')
-        shell: bash
-        run: ./scripts/setup_web.sh
+          pixi-version: v0.6.0
 
-      # The first steps of setup_web.sh, for Windows:
-      - name: Install wasm32 cargo target for building the web-viewer Wasm on windows
-        if: inputs.PLATFORM == 'windows'
+      - name: Build rerun-cli
         shell: bash
-        run: rustup target add wasm32-unknown-unknown
-
-      # The last step of setup_web.sh, for Windows.
-      # Since 'winget' is not available within the GitHub runner, we download the package directly:
-      # See: https://github.com/marketplace/actions/engineerd-configurator
-      - name: Install binaryen for building the web-viewer Wasm on windows
-        if: inputs.PLATFORM == 'windows'
-        uses: engineerd/configurator@v0.0.9
-        with:
-          name: "wasm-opt.exe"
-          url: "https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-windows.tar.gz"
-          pathInArchive: "binaryen-version_111/bin/wasm-opt.exe"
-
-      - name: Build rerun-cli (release)
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --locked -p rerun-cli --no-default-features --features native_viewer,web_viewer --release --target ${{ needs.set-config.outputs.TARGET }}
+        run: |
+          pixi run cargo build \
+            --locked \
+            -p rerun-cli \
+            --no-default-features \
+            --features native_viewer,web_viewer \
+            --release \
+            --target ${{ needs.set-config.outputs.TARGET }}
 
       - id: "auth"
         uses: google-github-actions/auth@v1
@@ -196,7 +160,8 @@ jobs:
         id: get-sha
         shell: bash
         run: |
-          echo "sha=$(echo ${{ inputs.RELEASE_COMMIT || github.sha }} | cut -c1-7)" >> "$GITHUB_OUTPUT"
+          full_commit="${{ inputs.RELEASE_COMMIT || ((github.event_name == 'pull_request' && github.event.pull_request.head.ref) || github.sha) }}"
+          echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT"
 
       - name: "Upload rerun-cli (commit)"
         uses: google-github-actions/upload-cloud-storage@v1
@@ -212,3 +177,4 @@ jobs:
           path: "./target/${{ needs.set-config.outputs.TARGET }}/release/${{ needs.set-config.outputs.BIN_NAME }}"
           destination: "rerun-builds/adhoc/${{inputs.ADHOC_NAME}}/rerun-cli/${{ inputs.PLATFORM }}"
           parent: false
+
diff --git a/.github/workflows/reusable_build_web_demo.yml b/.github/workflows/reusable_build_web_demo.yml
@@ -9,12 +9,6 @@ on:
       WHEEL_ARTIFACT_NAME:
         required: true
         type: string
-      # Each example in the built app contains a link to its source code.
-      # This determines the target commit to link to, and by default
-      # it is the current commit (github.sha)
-      SOURCE_LINK_COMMIT_OVERRIDE:
-        required: false
-        type: string
 
 concurrency:
   group: ${{ inputs.CONCURRENCY }}-build-web-demo
@@ -59,39 +53,37 @@ jobs:
           name: ${{ inputs.WHEEL_ARTIFACT_NAME }}
           path: wheel
 
-      - name: Install dependencies for examples/python
-        run: |
-          pip install -r scripts/ci/requirements.txt
-          pip install -r scripts/ci/requirements-web-demo.txt
+      - uses: prefix-dev/setup-pixi@v0.3.0
+        with:
+          pixi-version: v0.6.0
 
-      - name: Install built wheel
+      - name: Install Python dependencies and wheel
         shell: bash
         run: |
-          pip uninstall rerun-sdk -y
-          pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2
-          pip install rerun-sdk --no-index --find-links wheel
+          pixi run pip install -r scripts/ci/requirements-web-demo.txt
+          pixi run pip uninstall rerun-sdk -y
+          pixi run pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2
+          pixi run pip install rerun-sdk --no-index --find-links wheel
 
       - name: Verify built wheel version
         shell: bash
         run: |
-          python3 -m rerun --version
-          which rerun
-          rerun --version
+          pixi run python3 -m rerun --version
+          pixi run which rerun
+          pixi run rerun --version
 
-      - name: Resolve source link commit hash
+      - name: Get sha
+        id: get-sha
+        shell: bash
         run: |
-          if [ -z "${{ inputs.SOURCE_LINK_COMMIT_OVERRIDE }}" ]; then
-            USED_SHA=${{ github.sha }}
-          else
-            USED_SHA=${{ inputs.SOURCE_LINK_COMMIT_OVERRIDE }}
-          fi
-          echo "SHORT_SHA=$(echo $USED_SHA | cut -c1-7)" >> $GITHUB_ENV
+          full_commit="${{ (github.event_name == 'pull_request' && github.event.pull_request.head.ref) || github.sha }}"
+          echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT"
 
       - name: Build web demo
         env:
-          COMMIT_HASH: ${{ env.SHORT_SHA }}
+          COMMIT_HASH: ${{ steps.get-sha.outputs.sha }}
         run: |
-          python3 scripts/ci/build_demo_app.py --skip-build
+          pixi run python3 scripts/ci/build_demo_app.py --skip-build
 
       - name: Upload web demo assets
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/reusable_build_wheels.yml b/.github/workflows/reusable_build_wheels.yml
@@ -123,64 +123,20 @@ jobs:
           workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
           service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
 
-      # The pip-cache setup logic doesn't work in the ubuntu docker container
-      # That's probably fine since we bake these deps into the container already
-      - name: Setup python
-        if: ${{ inputs.PLATFORM != 'linux' }}
-        uses: actions/setup-python@v4
+      - uses: prefix-dev/setup-pixi@v0.3.0
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
-          cache: "pip"
-          cache-dependency-path: "scripts/ci/requirements.txt"
-
-      # These should already be in the docker container, but run for good measure. A no-op install
-      # should be fast, and this way things don't break if we add new packages without rebuilding
-      # docker
-      - run: pip install -r scripts/ci/requirements.txt
-
-      # We have a nice script for that: ./scripts/setup_web.sh
-      # Unfortunately, we can't run that on Windows, because Windows doesn't come with
-      # a package manager like grown-up OSes do (at least not the CI version of Windows).
-      # Also we can't run it on linux because the 20.04 Docker container will install
-      # an old version of binaryen/wasm-opt that barfs on the `--fast-math` flag
-      # So we only run the script on macos, and then on Windows we do the parts of the script manually.
-      # On ubuntu, the correct packages are pre-installed in our docker container.
-
-      - name: Install prerequisites for building the web-viewer Wasm (non-Windows)
-        if: (inputs.PLATFORM == 'macos-intel') || (inputs.PLATFORM == 'macos-arm')
-        shell: bash
-        run: ./scripts/setup_web.sh
-
-      # The first steps of setup_web.sh, for Windows:
-      - name: Install wasm32 cargo target for building the web-viewer Wasm on windows
-        if: inputs.PLATFORM == 'windows'
-        shell: bash
-        run: rustup target add wasm32-unknown-unknown
-
-      # The last step of setup_web.sh, for Windows.
-      # Since 'winget' is not available within the GitHub runner, we download the package directly:
-      # See: https://github.com/marketplace/actions/engineerd-configurator
-      - name: Install binaryen for building the web-viewer Wasm on windows
-        if: inputs.PLATFORM == 'windows'
-        uses: engineerd/configurator@v0.0.9
-        with:
-          name: "wasm-opt.exe"
-          url: "https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-windows.tar.gz"
-          pathInArchive: "binaryen-version_111/bin/wasm-opt.exe"
+          pixi-version: v0.6.0
 
       - name: Build Wheel
-        uses: PyO3/maturin-action@v1
-        with:
-          maturin-version: "0.14.17"
-          manylinux: manylinux_2_31
-          container: off
-          command: build
-          args: |
-            --manifest-path rerun_py/Cargo.toml
-            --release
-            --target ${{ needs.set-config.outputs.TARGET }}
+        run: |
+          pixi run pip install -r rerun_py/requirements-build.txt
+          pixi run maturin build \
+            --manylinux 2_31 \
+            --release \
+            --manifest-path rerun_py/Cargo.toml \
+            --target ${{ needs.set-config.outputs.TARGET }} \
+            --out dist \
             ${{ inputs.MATURIN_FEATURE_FLAGS }}
-            --out dist
 
       - name: Save wheel artifact
         if: ${{ inputs.WHEEL_ARTIFACT_NAME != '' }}

diff --git a/.github/workflows/reusable_bundle_and_upload_rerun_cpp.yml b/.github/workflows/reusable_bundle_and_upload_rerun_cpp.yml
@@ -49,7 +49,9 @@ jobs:
         id: get-sha
         shell: bash
         run: |
-          echo "sha=$(echo ${{ inputs.RELEASE_COMMIT || github.sha }} | cut -c1-7)" >> "$GITHUB_OUTPUT"
+          full_commit="${{ inputs.RELEASE_COMMIT || ((github.event_name == 'pull_request' && github.event.pull_request.head.ref) || github.sha) }}"
+          echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT"
 
       - name: "Bundle and upload rerun_cpp_sdk.zip"
         run: python3 ./scripts/ci/bundle_and_upload_rerun_cpp.py --git-hash ${{ steps.get-sha.outputs.sha }} --platform-filter=${{ inputs.PLATFORM_FILTER }}
+
diff --git a/.github/workflows/reusable_checks_cpp.yml b/.github/workflows/reusable_checks_cpp.yml
@@ -73,7 +73,6 @@ jobs:
       - uses: prefix-dev/setup-pixi@v0.3.0
         with:
           pixi-version: v0.6.0
-          cache: true
 
       - name: Set up Rust
         uses: ./.github/actions/setup-rust

diff --git a/.github/workflows/reusable_deploy_docs.yml b/.github/workflows/reusable_deploy_docs.yml
@@ -191,7 +191,6 @@ jobs:
       - uses: prefix-dev/setup-pixi@v0.3.0
         with:
           pixi-version: v0.6.0
-          cache: true
 
       - name: Doxygen C++ docs
         run: pixi run cpp-docs

diff --git a/.github/workflows/reusable_publish_web.yml b/.github/workflows/reusable_publish_web.yml
@@ -74,8 +74,9 @@ jobs:
           workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
           service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
 
-      - name: Install Python CI dependencies
-        run: pip install -r scripts/ci/requirements.txt
+      - uses: prefix-dev/setup-pixi@v0.3.0
+        with:
+          pixi-version: v0.6.0
 
       # built by `reusable_build_and_publish_wheels`
       - name: Download Wheel
@@ -90,40 +91,31 @@ jobs:
           name: ${{ inputs.rrd-artifact-name }}
           path: rrd
 
-      - name: Install dependencies for examples/python
-        run: |
-          pip install -r scripts/ci/requirements.txt
-          pip install -r scripts/ci/requirements-web-demo.txt
-
-      - name: Install built wheel
+      - name: Install Python dependencies and wheel
         shell: bash
         run: |
-          pip uninstall rerun-sdk -y
-          pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2
-          pip install rerun-sdk --no-index --find-links wheel
+          pixi run pip install -r scripts/ci/requirements.txt
+          pixi run pip install -r scripts/ci/requirements-web-demo.txt
+          pixi run pip uninstall rerun-sdk -y
+          pixi run pip install deprecated numpy>=1.23 pyarrow==10.0.1 pytest==7.1.2
+          pixi run pip install rerun-sdk --no-index --find-links wheel
 
       - name: Installed wheel version
         shell: bash
         run: |
-          python3 -m rerun --version
-          which rerun
-          rerun --version
-
-      - name: Install Wasm tools
-        shell: bash
-        run: ./scripts/setup_web.sh
+          pixi run python3 -m rerun --version
+          pixi run which rerun
+          pixi run rerun --version
 
       - name: Build app.rerun.io
-        uses: actions-rs/cargo@v1
-        with:
-          command: run
-          args: --locked -p re_build_web_viewer -- --release
+        run: |
+          pixi run cargo run --locked -p re_build_web_viewer -- --release
 
       - name: Build web demo
         env:
           COMMIT_HASH: ${{ needs.get-commit-sha.outputs.short-sha }}
         run: |
-          python3 scripts/ci/build_demo_app.py --skip-build
+          pixi run python3 scripts/ci/build_demo_app.py --skip-build
 
       - name: Upload .rrd for app.rerun.io
         uses: google-github-actions/upload-cloud-storage@v1

diff --git a/.github/workflows/reusable_upload_wheels.yml b/.github/workflows/reusable_upload_wheels.yml
@@ -84,7 +84,8 @@ jobs:
         id: get-sha
         shell: bash
         run: |
-          echo "sha=$(echo ${{ inputs.RELEASE_COMMIT || github.sha }} | cut -c1-7)" >> "$GITHUB_OUTPUT"
+          full_commit="${{ inputs.RELEASE_COMMIT || ((github.event_name == 'pull_request' && github.event.pull_request.head.ref) || github.sha) }}"
+          echo "sha=$(echo $full_commit | cut -c1-7)" >> "$GITHUB_OUTPUT"
 
       - name: "Upload Wheel"
         uses: google-github-actions/upload-cloud-storage@v1

diff --git a/ci_docker/Dockerfile b/ci_docker/Dockerfile
@@ -84,6 +84,12 @@ RUN set -eux; \
     cargo --version; \
     rustc --version;
 
+# Install stuff from `rust-toolchain`
+ADD rust-toolchain rust-toolchain
+# `show` is currently the only way to install whatever is in `rust-toolchain`
+# This is subject to change, but there is no alternative: https://github.com/rust-lang/rustup/issues/1397
+RUN rustup show
+
 # Install some cargo tools we know we'll always need
 # We can't do this until after we've installed rust / cargo above
 # Latest cargo-deny requires a rust update!
@@ -97,10 +103,6 @@ RUN pip install -r requirements-build.txt
 ADD rerun_py/requirements-lint.txt requirements-lint.txt
 RUN pip install -r requirements-lint.txt
 
-# Install tools from setup_web.sh
-ADD scripts/setup_web.sh setup_web.sh
-# Installing binaryen is going to cause this script to fail, but that's ok -- we install it manually
-RUN ./setup_web.sh || true
 # Note: We need a more modern binaryen than ships on ubuntu-20.4, so we pull it from github
 RUN curl -L https://github.com/WebAssembly/binaryen/releases/download/version_112/binaryen-version_112-x86_64-linux.tar.gz | tar xzk --strip-components 1