CI: Rerun CLI as a release asset

.github/workflows/on_pull_request.yml

@@ -44,6 +44,16 @@ jobs:
       UPLOAD_COMMIT_OVERRIDE: ${{ github.event.pull_request.head.sha }}
     secrets: inherit
 
+  build-rerun-cli-and-upload:
+    name: "Build & Upload rerun-cli (Linux x64)"
+    if: github.event.pull_request.head.repo.owner.login == 'rerun-io'
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
+    with:
+      CONCURRENCY: pr-${{ github.event.pull_request.number }}
+      PLATFORM: linux
+      UPLOAD_COMMIT_OVERRIDE: ${{ github.event.pull_request.head.sha }}
+    secrets: inherit
+
   bundle-and-upload-rerun_cpp:
     name: "Bundle and upload rerun_cpp"
     needs: [build-rerun_c-and-upload]

.github/workflows/on_push_main.yml

@@ -129,6 +129,42 @@ jobs:
       PLATFORM: windows
     secrets: inherit
 
+  build-rerun-cli-and-upload-linux:
+    needs: [checks]
+    name: "Linux-x64: Build & Upload rerun-cli"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
+    with:
+      CONCURRENCY: push-linux-${{ github.ref_name }}
+      PLATFORM: linux
+    secrets: inherit
+
+  build-rerun-cli-and-upload-macos-intel:
+    needs: [checks]
+    name: "Mac-Intel: Build & Upload rerun-cli"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
+    with:
+      CONCURRENCY: push-macos-intel-${{ github.ref_name }}
+      PLATFORM: macos-intel
+    secrets: inherit
+
+  build-rerun-cli-and-upload-macos-arm:
+    needs: [checks]
+    name: "Mac-Arm: Build & Upload rerun-cli"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
+    with:
+      CONCURRENCY: push-macos-arm-${{ github.ref_name }}
+      PLATFORM: macos-arm
+    secrets: inherit
+
+  build-rerun-cli-and-upload-windows:
+    needs: [checks]
+    name: "Windows-x64: Build & Upload rerun-cli"
+    uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml
+    with:
+      CONCURRENCY: push-windows-${{ github.ref_name }}
+      PLATFORM: windows
+    secrets: inherit
+
   build-linux:
     needs: [checks]
     name: "Linux: Build/Test Wheels"

.github/workflows/reusable_build_and_upload_rerun_cli.yml

@@ -0,0 +1,194 @@
+name: Reusable Rerun CLI build & upload
+
+on:
+  workflow_call:
+    inputs:
+      CONCURRENCY:
+        required: true
+        type: string
+      PLATFORM:
+        required: true
+        type: string
+      RELEASE_VERSION:
+        required: false
+        type: string
+        default: "prerelease"
+      UPLOAD_COMMIT_OVERRIDE:
+        required: false
+        type: string
+        default: ""
+      UPLOAD_COMMIT:
+        required: false
+        type: boolean
+        default: true
+      ADHOC_NAME:
+        required: false
+        type: string
+        default: ""
+
+  workflow_dispatch:
+    inputs:
+      ADHOC_NAME:
+        required: true
+        type: string
+        description: "Name of the adhoc build, used for upload directory"
+      PLATFORM:
+        type: choice
+        options:
+          - linux
+          - windows
+          - macos-arm
+          - macos-intel
+        description: "Platform to build for"
+        required: true
+      CONCURRENCY:
+        required: false
+        type: string
+        default: "adhoc"
+      RELEASE_VERSION:
+        required: false
+        type: string
+        default: "prerelease"
+      UPLOAD_COMMIT_OVERRIDE:
+        required: false
+        type: string
+        default: ""
+      UPLOAD_COMMIT:
+        required: false
+        type: boolean
+        default: true
+
+concurrency:
+  group: ${{ inputs.CONCURRENCY }}-build-rerun-cli
+  cancel-in-progress: true
+
+env:
+  # web_sys_unstable_apis is required to enable the web_sys clipboard API which egui_web uses
+  # https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html
+  # https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html
+  RUSTFLAGS: --cfg=web_sys_unstable_apis --deny warnings
+
+  # See https://github.com/ericseppanen/cargo-cranky/issues/8
+  RUSTDOCFLAGS: --deny warnings --deny rustdoc::missing_crate_level_docs
+
+permissions:
+  contents: "read"
+  id-token: "write"
+
+jobs:
+  set-config:
+    name: Set Config (${{ inputs.PLATFORM }})
+    runs-on: ubuntu-latest-16-cores
+    outputs:
+      RUNNER: ${{ steps.set-config.outputs.runner }}
+      TARGET: ${{ steps.set-config.outputs.target }}
+      CONTAINER: ${{ steps.set-config.outputs.container }}
+      BIN_NAME: ${{ steps.set-config.outputs.bin_name }}
+    steps:
+      - name: Set runner and target based on platform
+        id: set-config
+        run: |
+          case "${{ inputs.PLATFORM }}" in
+            linux)
+              runner="ubuntu-latest-16-cores"
+              target="x86_64-unknown-linux-gnu"
+              container="{'image': 'rerunio/ci_docker:0.10.0'}"
+              bin_name="rerun"
+              ;;
+            windows)
+              runner="windows-latest-8-cores"
+              target="x86_64-pc-windows-msvc"
+              container="null"
+              bin_name="rerun.exe"
+              ;;
+            macos-arm)
+              runner="macos-latest"
+              target="aarch64-apple-darwin"
+              container="null"
+              bin_name="rerun"
+              ;;
+            macos-intel)
+              runner="macos-latest"
+              target="x86_64-apple-darwin"
+              container="null"
+              bin_name="rerun"
+              ;;
+            *) echo "Invalid platform" && exit 1 ;;
+          esac
+          echo "runner=$runner" >> "$GITHUB_OUTPUT"
+          echo "target=$target" >> "$GITHUB_OUTPUT"
+          echo "container=$container" >> "$GITHUB_OUTPUT"
+          echo "bin_name=$bin_name" >> "$GITHUB_OUTPUT"
+
+  build-rerun-cli:
+    name: Build rerun-cli (${{ needs.set-config.outputs.RUNNER }})
+
+    needs: [set-config]
+
+    runs-on: ${{ needs.set-config.outputs.RUNNER }}
+    container: ${{ fromJson(needs.set-config.outputs.CONTAINER) }}
+
+    steps:
+      - name: Show context
+        run: |
+          echo "GITHUB_CONTEXT": $GITHUB_CONTEXT
+          echo "JOB_CONTEXT": $JOB_CONTEXT
+          echo "INPUTS_CONTEXT": $INPUTS_CONTEXT
+          echo "ENV_CONTEXT": $ENV_CONTEXT
+        env:
+          ENV_CONTEXT: ${{ toJson(env) }}
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+          JOB_CONTEXT: ${{ toJson(job) }}
+          INPUTS_CONTEXT: ${{ toJson(inputs) }}
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.ref) || '' }}
+
+      - name: Set up Rust
+        uses: ./.github/actions/setup-rust
+        with:
+          cache_key: "build-${{ inputs.PLATFORM }}"
+          save_cache: false
+          workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
+          targets: ${{ needs.set-config.outputs.TARGET }}
+
+      - name: Build rerun-cli (release)
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --locked -p rerun-cli --no-default-features --features native_viewer,web_viewer --release --target ${{ needs.set-config.outputs.TARGET }}
+
+      - id: "auth"
+        uses: google-github-actions/auth@v1
+        with:
+          workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
+
+      - name: Add SHORT_SHA env property with commit short sha
+        shell: bash
+        id: "short-sha"
+        run: |
+          if [ -z "${{ inputs.UPLOAD_COMMIT_OVERRIDE }}" ]; then
+            USED_SHA=${{ github.sha }}
+          else
+            USED_SHA=${{ inputs.UPLOAD_COMMIT_OVERRIDE }}
+          fi
+          echo "SHORT_SHA=$(echo $USED_SHA | cut -c1-7)" >> $GITHUB_OUTPUT
+
+      - name: "Upload rerun-cli (commit)"
+        if: ${{ inputs.UPLOAD_COMMIT }}
+        uses: google-github-actions/upload-cloud-storage@v1
+        with:
+          path: "./target/${{ needs.set-config.outputs.TARGET }}/release/${{ needs.set-config.outputs.BIN_NAME }}"
+          destination: "rerun-builds/commit/${{ steps.short-sha.outputs.SHORT_SHA }}/rerun-cli/${{ inputs.PLATFORM }}"
+          parent: false
+
+      - name: "Upload rerun-cli (adhoc)"
+        if: ${{ inputs.ADHOC_NAME != '' }}
+        uses: google-github-actions/upload-cloud-storage@v1
+        with:
+          path: "./target/${{ needs.set-config.outputs.TARGET }}/release/${{ needs.set-config.outputs.BIN_NAME }}"
+          destination: "rerun-builds/adhoc/${{inputs.ADHOC_NAME}}/rerun-cli/${{ inputs.PLATFORM }}"
+          parent: false

scripts/ci/sync_release_assets.py

@@ -26,7 +26,13 @@
 
 
 def fetch_binary_assets(
-    commit: str, *, do_wheels: bool = True, do_rerun_c: bool = True, do_rerun_cpp_sdk: bool = True
+    tag: str,
+    commit: str,
+    *,
+    do_wheels: bool = True,
+    do_rerun_c: bool = True,
+    do_rerun_cpp_sdk: bool = True,
+    do_rerun_cli: bool = True,
 ) -> Assets:
     """Given a release ID, fetches all associated binary assets from our cloud storage (build.rerun.io)."""
     assets = dict()
@@ -39,33 +45,48 @@ def fetch_binary_assets(
     print(f"  - wheels: {do_wheels}")
     print(f"  - C libs: {do_rerun_c}")
     print(f"  - C++ uber SDK: {do_rerun_cpp_sdk}")
+    print(f"  - CLI (Viewer): {do_rerun_cli}")
 
     # Python wheels
     if do_wheels:
         wheel_blobs = list(bucket.list_blobs(prefix=f"commit/{commit_short}/wheels"))
         for blob in [bucket.get_blob(blob.name) for blob in wheel_blobs if blob.name.endswith(".whl")]:
             if blob is not None and blob.name is not None:
                 name = blob.name.split("/")[-1]
+
+                if "macosx" in name:
+                    if "x86_64" in name:
+                        name = f"rerun_sdk-{tag}-aarch64-apple-darwin.whl"
+                    if "arm64" in name:
+                        name = f"rerun_sdk-{tag}-x86_64-apple-darwin.whl"
+
+                if "manylinux_2_31_x86_64" in name:
+                    if "x86_64" in name:
+                        name = f"rerun_sdk-{tag}-x86_64-unknown-linux-gnu.whl"
+
+                if "win_amd64" in name:
+                    name = f"rerun_sdk-{tag}-x86_64-pc-windows-msvc.whl"
+
                 print(f"    Found Python wheel: {name} ")
                 assets[name] = blob
 
     # rerun_c
     if do_rerun_c:
         rerun_c_blobs = [
             (
-                "librerun_c.x86_64-pc-windows-msvc.lib",
+                f"rerun_c-{tag}-x86_64-pc-windows-msvc.lib",
                 bucket.get_blob(f"commit/{commit_short}/rerun_c/windows/rerun_c.lib"),
             ),
             (
-                "librerun_c.x86_64-unknown-linux-gnu.a",
+                f"librerun_c-{tag}-x86_64-unknown-linux-gnu.a",
                 bucket.get_blob(f"commit/{commit_short}/rerun_c/linux/librerun_c.a"),
             ),
             (
-                "librerun_c.aarch64-apple-darwin.a",
+                f"librerun_c-{tag}-aarch64-apple-darwin.a",
                 bucket.get_blob(f"commit/{commit_short}/rerun_c/macos-arm/librerun_c.a"),
             ),
             (
-                "librerun_c.x86_64-apple-darwin.a",
+                f"librerun_c-{tag}-x86_64-apple-darwin.a",
                 bucket.get_blob(f"commit/{commit_short}/rerun_c/macos-intel/librerun_c.a"),
             ),
         ]
@@ -82,14 +103,33 @@ def fetch_binary_assets(
                 name = blob.name.split("/")[-1]
                 print(f"    Found Rerun cross-platform bundle: {name}")
                 assets[name] = blob
+                # NOTE: Want a versioned one too.
+                assets[f"rerun_cpp_sdk-{tag}-multiplatform.zip"] = blob
 
-    # rerun_cpp_sdk
-    rerun_cpp_sdk_blob = bucket.get_blob(f"commit/{commit_short}/rerun_cpp_sdk.zip")
-    for blob in [rerun_cpp_sdk_blob]:
-        if blob is not None and blob.name is not None:
-            name = blob.name.split("/")[-1]
-            print(f"    Found Rerun cross-platform bundle: {name} ({blob.size} bytes)")
-            assets[name] = blob
+    # rerun-cli
+    if do_rerun_cli:
+        rerun_cli_blobs = [
+            (
+                f"rerun-cli-{tag}-x86_64-pc-windows-msvc.exe",
+                bucket.get_blob(f"commit/{commit_short}/rerun-cli/windows/rerun.exe"),
+            ),
+            (
+                f"rerun-cli-{tag}-x86_64-unknown-linux-gnu",
+                bucket.get_blob(f"commit/{commit_short}/rerun-cli/linux/rerun"),
+            ),
+            (
+                f"rerun-cli-{tag}-aarch64-apple-darwin",
+                bucket.get_blob(f"commit/{commit_short}/rerun-cli/macos-arm/rerun"),
+            ),
+            (
+                f"rerun-cli-{tag}-x86_64-apple-darwin",
+                bucket.get_blob(f"commit/{commit_short}/rerun-cli/macos-intel/rerun"),
+            ),
+        ]
+        for name, blob in rerun_cli_blobs:
+            if blob is not None:
+                print(f"    Found Rerun CLI binary: {name}")
+                assets[name] = blob
 
     return assets
 
@@ -134,6 +174,7 @@ def main() -> None:
     parser.add_argument("--no-wheels", action="store_true", help="Don't upload Python wheels")
     parser.add_argument("--no-rerun-c", action="store_true", help="Don't upload C libraries")
     parser.add_argument("--no-rerun-cpp-sdk", action="store_true", help="Don't upload C++ uber SDK")
+    parser.add_argument("--no-rerun-cli", action="store_true", help="Don't upload CLI")
     args = parser.parse_args()
 
     gh = Github(args.github_token, timeout=args.github_timeout)
@@ -146,10 +187,12 @@ def main() -> None:
     )
 
     assets = fetch_binary_assets(
+        release.tag_name,
         commit.sha,
         do_wheels=not args.no_wheels,
         do_rerun_c=not args.no_rerun_c,
         do_rerun_cpp_sdk=not args.no_rerun_cpp_sdk,
+        do_rerun_cli=not args.no_rerun_cli,
     )
 
     if args.remove: