Make PR Validation job more parallelized - add some rust caching

.github/workflows/Benchmarks.yml

@@ -6,7 +6,6 @@ on:
   workflow_call: # This is called from CreateRelease.yml
 
 permissions:
-  id-token: write
   contents: read
 
 jobs:
@@ -40,20 +39,14 @@ jobs:
       - name: Download Rust Guest Binaries
         uses: actions/download-artifact@v6
         with:
-          name: rust-guest-binaries-release
-          path: ./downloaded-rust-guest-binaries-release
+          name: rust-guests-release
+          path: src/tests/rust_guests/bin/release/
 
       - name: Download C Guest Binaries
         uses: actions/download-artifact@v6
         with:
-          name: c-guest-binaries-release
-          path: ./downloaded-c-guest-binaries-release
-
-      - name: Copy Guest Binaries
-        run: |
-          cp ./downloaded-rust-guest-binaries-release/simpleguest ./src/tests/rust_guests/bin/release/simpleguest
-          cp ./downloaded-rust-guest-binaries-release/dummyguest ./src/tests/rust_guests/bin/release/dummyguest
-          cp ./downloaded-c-guest-binaries-release/simpleguest ./src/tests/c_guests/bin/release/simpleguest
+          name: c-guests-release
+          path: src/tests/c_guests/bin/release/
 
       ### Benchmarks ###
       - name: Fetch tags

.github/workflows/CargoPublish.yml

@@ -18,7 +18,6 @@ on:
 
 permissions:
   contents: read
-  id-token: write
 
 jobs:
   publish-hyperlight-packages:

.github/workflows/CreateRelease.yml

@@ -9,7 +9,6 @@ on:
 
 permissions:
   contents: write
-  id-token: write
   issues: read
 
 jobs:
@@ -64,17 +63,18 @@ jobs:
       - name: Build Release
         run: cargo build --verbose --release
 
-  build-guest-binaries:
-    uses: ./.github/workflows/dep_build_guest_binaries.yml
+  build-guests:
+    uses: ./.github/workflows/dep_build_guests.yml
     secrets: inherit
     needs: [release-blocker-check]
+    with:
+      config: release
 
   benchmarks:
-    needs: [build-guest-binaries]
+    needs: [build-guests]
     uses: ./.github/workflows/Benchmarks.yml
     secrets: inherit
     permissions:
-      id-token: write
       contents: read
 
   cargo-publish:
@@ -84,7 +84,6 @@ jobs:
       dry_run: false
     secrets: inherit
     permissions:
-      id-token: write
       contents: read
 
   publish:
@@ -140,9 +139,11 @@ jobs:
 
       - name: Archive benchmarks
         run: |
-          # windows
+          # windows (hyperv = Server 2022, hyperv-ws2025 = Server 2025)
           tar -zcvf benchmarks_Windows_hyperv_amd.tar.gz benchmarks_Windows_hyperv_amd
           tar -zcvf benchmarks_Windows_hyperv_intel.tar.gz benchmarks_Windows_hyperv_intel
+          tar -zcvf benchmarks_Windows_hyperv-ws2025_amd.tar.gz benchmarks_Windows_hyperv-ws2025_amd
+          tar -zcvf benchmarks_Windows_hyperv-ws2025_intel.tar.gz benchmarks_Windows_hyperv-ws2025_intel
           # kvm
           tar -zcvf benchmarks_Linux_kvm_amd.tar.gz benchmarks_Linux_kvm_amd
           tar -zcvf benchmarks_Linux_kvm_intel.tar.gz benchmarks_Linux_kvm_intel
@@ -168,6 +169,8 @@ jobs:
             gh release create v${{ env.HYPERLIGHT_VERSION }} -t "Release v${{ env.HYPERLIGHT_VERSION }}" --notes-file RELEASE_NOTES.md \
             benchmarks_Windows_hyperv_amd.tar.gz \
             benchmarks_Windows_hyperv_intel.tar.gz \
+            benchmarks_Windows_hyperv-ws2025_amd.tar.gz \
+            benchmarks_Windows_hyperv-ws2025_intel.tar.gz \
             benchmarks_Linux_kvm_amd.tar.gz \
             benchmarks_Linux_kvm_intel.tar.gz \
             benchmarks_Linux_mshv3_amd.tar.gz \
@@ -184,6 +187,8 @@ jobs:
             gh release create dev-latest -t "Latest prerelease from main branch" --notes-file RELEASE_NOTES.md --latest=false -p \
             benchmarks_Windows_hyperv_amd.tar.gz \
             benchmarks_Windows_hyperv_intel.tar.gz \
+            benchmarks_Windows_hyperv-ws2025_amd.tar.gz \
+            benchmarks_Windows_hyperv-ws2025_intel.tar.gz \
             benchmarks_Linux_kvm_amd.tar.gz \
             benchmarks_Linux_kvm_intel.tar.gz \
             benchmarks_Linux_mshv3_amd.tar.gz \

.github/workflows/Fuzzing.yml

@@ -6,7 +6,6 @@ on:
   workflow_dispatch: # Allow manual triggering
 
 permissions:
-  id-token: write
   contents: read
 
 jobs:

.github/workflows/RustNightly.yml

@@ -9,24 +9,119 @@ on:
     - cron: '0 0 */2 * *'
 
 permissions:
-  id-token: write
   contents: read
 
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: full
+
+defaults:
+  run:
+    shell: bash
+
 jobs:
+  # Nightly musl cross-compilation builds
+  # This is a self-contained job since musl builds are a special case
+  # that require TARGET_TRIPLE for cross-compilation
   musl:
+    timeout-minutes: 60
     strategy:
       fail-fast: true
       matrix:
         hypervisor: [kvm, mshv3]
         cpu: [amd, intel]
         config: [debug, release]
-    uses: ./.github/workflows/dep_rust.yml
-    secrets: inherit
-    with: 
-      hypervisor: ${{ matrix.hypervisor }}
-      cpu: ${{ matrix.cpu }}
-      config: ${{ matrix.config }}
-      target_triple: x86_64-unknown-linux-musl
+    runs-on: ${{ fromJson(
+        format('["self-hosted", "Linux", "X64", "1ES.Pool=hld-{0}-{1}"]', 
+          matrix.hypervisor == 'mshv3' && 'azlinux3-mshv' || matrix.hypervisor, 
+          matrix.cpu)) }}
+    env:
+      TARGET_TRIPLE: x86_64-unknown-linux-musl
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: hyperlight-dev/ci-setup-workflow@v1.8.0
+        with:
+          rust-toolchain: "1.89"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Fix cargo home permissions
+        run: |
+          sudo chown -R $(id -u):$(id -g) /opt/cargo || true
+
+      # cargo-hyperlight builds a custom sysroot for x86_64-hyperlight-none target.
+      # rust-cache cleans "anything not a dependency" from target dirs, removing the sysroot.
+      # We cache sysroot separately to avoid rebuilding it (~10s) on every run.
+      - name: Sysroot cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            src/tests/rust_guests/simpleguest/target/sysroot
+            src/tests/rust_guests/dummyguest/target/sysroot
+            src/tests/rust_guests/witguest/target/sysroot
+          key: sysroot-linux-${{ matrix.config }}-${{ hashFiles('rust-toolchain.toml') }}
+
+      - name: Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: "nightly-${{ matrix.config }}"
+          cache-on-failure: "true"
+          workspaces: |
+            . -> target
+            src/tests/rust_guests/simpleguest -> target
+            src/tests/rust_guests/dummyguest -> target
+            src/tests/rust_guests/witguest -> target
+
+      - name: Build and move Rust guests
+        run: |
+          just build-rust-guests ${{ matrix.config }}
+          just move-rust-guests ${{ matrix.config }}
+
+      - name: Build C guests
+        run: |
+          just build-c-guests ${{ matrix.config }}
+          just move-c-guests ${{ matrix.config }}
+
+      - name: Build
+        run: just build ${{ matrix.config }}
+
+      - name: Run Miri tests
+        run: just miri-tests
+
+      - name: Run Rust tests
+        run: |
+          # with default features
+          just test ${{ matrix.config }}
+
+          # with only one driver enabled
+          just test ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || 'kvm' }}
+
+          # make sure certain cargo features compile
+          just check
+
+          # without any features
+          just test-compilation-no-default-features ${{ matrix.config }}
+
+      - name: Run Rust examples
+        env:
+          RUST_LOG: debug
+        run: just run-rust-examples-linux ${{ matrix.config }}
+
+      - name: Run Rust Gdb tests
+        env:
+          RUST_LOG: debug
+        run: just test-rust-gdb-debugging ${{ matrix.config }}
+
+      - name: Run Rust Crashdump tests
+        env:
+          RUST_LOG: debug
+        run: just test-rust-crashdump ${{ matrix.config }}
+
+      - name: Run Rust Tracing tests
+        env:
+          RUST_LOG: debug
+        run: just test-rust-tracing ${{ matrix.config }}
 
   notify-failure:
     runs-on: ubuntu-latest

.github/workflows/ValidatePullRequest.yml

@@ -14,7 +14,6 @@ concurrency:
   cancel-in-progress: true    
 
 permissions:
-  id-token: write
   contents: write
 
 jobs:
@@ -41,31 +40,91 @@ jobs:
             return all_file_count === docs_file_count;
           result-encoding: string
 
-  rust:
+  # Build guests once, upload as artifacts for other jobs to download
+  build-guests:
+    needs: docs-pr
+    strategy:
+      fail-fast: true
+      matrix:
+        config: [debug, release]
+    uses: ./.github/workflows/dep_build_guests.yml
+    secrets: inherit
+    with:
+      docs_only: ${{ needs.docs-pr.outputs.docs-only }}
+      config: ${{ matrix.config }}
+
+  # Code checks (fmt, clippy, MSRV) - runs in parallel with build-guests
+  code-checks:
+    needs: docs-pr
+    uses: ./.github/workflows/dep_code_checks.yml
+    secrets: inherit
+    with:
+      docs_only: ${{ needs.docs-pr.outputs.docs-only }}
+
+  # Build and test - needs guest artifacts
+  build-test:
     needs:
       - docs-pr
+      - build-guests
     strategy:
       fail-fast: true
       matrix:
         hypervisor: [hyperv, 'hyperv-ws2025', mshv3, kvm]
         cpu: [amd, intel]
         config: [debug, release]
-    uses: ./.github/workflows/dep_rust.yml
+    uses: ./.github/workflows/dep_build_test.yml
     secrets: inherit
     with: 
-      docs_only: ${{needs.docs-pr.outputs.docs-only}}
+      docs_only: ${{ needs.docs-pr.outputs.docs-only }}
       hypervisor: ${{ matrix.hypervisor }}
       cpu: ${{ matrix.cpu }}
       config: ${{ matrix.config }}
 
+  # Run examples - needs guest artifacts, runs in parallel with build-test
+  run-examples:
+    needs:
+      - docs-pr
+      - build-guests
+    strategy:
+      fail-fast: true
+      matrix:
+        hypervisor: [hyperv, 'hyperv-ws2025', mshv3, kvm]
+        cpu: [amd, intel]
+        config: [debug, release]
+    uses: ./.github/workflows/dep_run_examples.yml
+    secrets: inherit
+    with: 
+      docs_only: ${{ needs.docs-pr.outputs.docs-only }}
+      hypervisor: ${{ matrix.hypervisor }}
+      cpu: ${{ matrix.cpu }}
+      config: ${{ matrix.config }}
+
+  # Run benchmarks - release only, needs guest artifacts, runs in parallel with build-test
+  benchmarks:
+    needs:
+      - docs-pr
+      - build-guests
+    strategy:
+      fail-fast: true
+      matrix:
+        hypervisor: [hyperv, 'hyperv-ws2025', mshv3, kvm]
+        cpu: [amd, intel]
+    uses: ./.github/workflows/dep_benchmarks.yml
+    secrets: inherit
+    with: 
+      docs_only: ${{ needs.docs-pr.outputs.docs-only }}
+      hypervisor: ${{ matrix.hypervisor }}
+      cpu: ${{ matrix.cpu }}
+
   fuzzing:
     needs:
       - docs-pr
+      - build-guests
     uses: ./.github/workflows/dep_fuzzing.yml
     with:
       targets: '["fuzz_host_print", "fuzz_guest_call", "fuzz_host_call", "fuzz_guest_estimate_trace_event", "fuzz_guest_trace"]' # Pass as a JSON array
       max_total_time: 300 # 5 minutes in seconds
-      docs_only: ${{needs.docs-pr.outputs.docs-only}}
+      docs_only: ${{ needs.docs-pr.outputs.docs-only }}
     secrets: inherit
 
   spelling:
@@ -85,21 +144,20 @@ jobs:
       run: ./dev/check-license-headers.sh
 
   # Gate PR merges on this specific "join-job" which requires all other
-  # jobs to run first. We need this job since we cannot gate on particular jobs
-  # in the workflow, since they can sometimes be skipped (e.g. if the PR only touches docs).
-  # This step fixes this issue by always running.
+  # jobs to run first.
   report-ci-status:
     needs:
       - docs-pr
-      - rust
+      - build-guests
+      - code-checks
+      - build-test
+      - run-examples
+      - benchmarks
       - fuzzing
       - spelling
       - license-headers
     if: always()
     runs-on: ubuntu-latest
     steps:
-      # Calculate the exit status of the whole CI workflow.
-      # If all dependent jobs were successful, this exits with 0 (and the outcome job continues successfully).
-      # If a some dependent job has failed, this exits with 1.
       - name: calculate the correct exit status
         run: jq --exit-status 'all(.result == "success" or .result == "skipped")' <<< '${{ toJson(needs) }}'