fix(sovereign-ci): self-heal broken sibling clone cache

.github/workflows/sovereign-ci.yml

@@ -4,7 +4,7 @@
 # Change once here → applies to all 38 repos instantly.
 #
 # Self-hosted jobs run inside the sovereign-ci container (built by forjar).
 # Each job gets an isolated filesystem — no shared ~/.rustup/, no race conditions.
 # Image: localhost:5000/sovereign-ci:stable (local registry on mac-server)
 # Rebuild: cd infra && make -f machines/intel/Makefile ci-image
 #
@@ -48,7 +48,7 @@
         default: false
         type: boolean
       extra_pkgs:
         description: 'Extra apt packages to install in container (e.g. python3-dev libclang-dev)'
         required: false
         default: ''
         type: string
@@ -73,7 +73,7 @@
 jobs:
   test:
     name: test
     runs-on: [self-hosted, clean-room]
     container:
       image: localhost:5000/sovereign-ci:stable@sha256:a7d47ef6e12e23c83075ceff4c41be8f34b00e68639a48bca9e41a2b2c8db80b
       # Phase 3 §5.3 — sccache rustc cache + /var/log/ci-metrics for F9 stats.
@@ -89,7 +89,7 @@
           persist-credentials: false
       - name: Install extra packages
         if: ${{ inputs.extra_pkgs != '' }}
         run: |
           apt-get update -qq && apt-get install -y -qq ${{ inputs.extra_pkgs }} 2>/dev/null || \
           sudo apt-get update -qq && sudo apt-get install -y -qq ${{ inputs.extra_pkgs }} 2>/dev/null || true
       - name: Checkout sibling repos (path deps)
@@ -106,12 +106,20 @@
             [ "$repo" = "${{ inputs.repo }}" ] && continue
             # Remove stale partial clones (file instead of dir)
             if [ -e "$repo" ] && [ ! -d "$repo" ]; then rm -f "$repo"; fi
+            # Validate cached clone. If HEAD missing or fetch fails, nuke + re-clone.
+            # Silent `|| true` on fetch previously masked broken caches (bashrs#197).
             if [ -d "$repo" ]; then
-              # Pull latest to avoid stale cached versions
-              git -C "$repo" fetch --depth 1 origin main 2>/dev/null && git -C "$repo" reset --hard FETCH_HEAD 2>/dev/null || true
-            else
+              if ! git -C "$repo" rev-parse HEAD >/dev/null 2>&1; then
+                echo "::warning::$repo cache invalid (no HEAD) — re-cloning"
+                rm -rf "$repo"
+              elif ! (git -C "$repo" fetch --depth 1 origin main 2>&1 && git -C "$repo" reset --hard FETCH_HEAD >/dev/null 2>&1); then
+                echo "::warning::$repo fetch failed — re-cloning"
+                rm -rf "$repo"
+              fi
+            fi
+            if [ ! -d "$repo" ]; then
               for attempt in 1 2 3; do
-                git clone --depth 1 --quiet "https://github.com/paiml/$repo.git" "$repo" 2>&1 && break
+                git clone --depth 1 "https://github.com/paiml/$repo.git" "$repo" 2>&1 && break
                 rm -rf "$repo"  # Clean failed clone before retry
                 echo "::warning::Retry $attempt for $repo clone"
                 sleep 2
@@ -129,7 +137,7 @@
           # Note: generated contract macros may have unused variables (provable-contracts#64).
           # This is handled by adding -A unused-variables to the clippy step.
       - name: Generate contract assertions (pv codegen)
         run: |
           # pv is baked into sovereign-ci:stable at /usr/local/cargo/bin/pv
           PV=""
           for candidate in /usr/local/cargo/bin/pv /usr/local/bin/pv; do
@@ -178,7 +186,7 @@
           REPO_NAME: ${{ inputs.repo }}
           RUSTC_WRAPPER: ${{ inputs.enable_sccache && 'sccache' || '' }}
           SCCACHE_DIR: ${{ inputs.enable_sccache && '/sccache' || '' }}
         run: |
           # Mark workspace as safe for git operations inside tests (dubious ownership in containers)
           git config --global --add safe.directory "$GITHUB_WORKSPACE"
           cargo test --lib $TEST_ARGS 2>&1 || \
@@ -193,7 +201,7 @@
 
   lint:
     name: lint
     runs-on: [self-hosted, clean-room]
     container:
       image: localhost:5000/sovereign-ci:stable@sha256:a7d47ef6e12e23c83075ceff4c41be8f34b00e68639a48bca9e41a2b2c8db80b
       # Phase 3 §5.3 — sccache rustc cache + /var/log/ci-metrics for F9 stats.
@@ -209,7 +217,7 @@
           persist-credentials: false
       - name: Install extra packages
         if: ${{ inputs.extra_pkgs != '' }}
         run: |
           apt-get update -qq && apt-get install -y -qq ${{ inputs.extra_pkgs }} 2>/dev/null || \
           sudo apt-get update -qq && sudo apt-get install -y -qq ${{ inputs.extra_pkgs }} 2>/dev/null || true
       - name: Checkout sibling repos (path deps)
@@ -226,12 +234,20 @@
             [ "$repo" = "${{ inputs.repo }}" ] && continue
             # Remove stale partial clones (file instead of dir)
             if [ -e "$repo" ] && [ ! -d "$repo" ]; then rm -f "$repo"; fi
+            # Validate cached clone. If HEAD missing or fetch fails, nuke + re-clone.
+            # Silent `|| true` on fetch previously masked broken caches (bashrs#197).
             if [ -d "$repo" ]; then
-              # Pull latest to avoid stale cached versions
-              git -C "$repo" fetch --depth 1 origin main 2>/dev/null && git -C "$repo" reset --hard FETCH_HEAD 2>/dev/null || true
-            else
+              if ! git -C "$repo" rev-parse HEAD >/dev/null 2>&1; then
+                echo "::warning::$repo cache invalid (no HEAD) — re-cloning"
+                rm -rf "$repo"
+              elif ! (git -C "$repo" fetch --depth 1 origin main 2>&1 && git -C "$repo" reset --hard FETCH_HEAD >/dev/null 2>&1); then
+                echo "::warning::$repo fetch failed — re-cloning"
+                rm -rf "$repo"
+              fi
+            fi
+            if [ ! -d "$repo" ]; then
               for attempt in 1 2 3; do
-                git clone --depth 1 --quiet "https://github.com/paiml/$repo.git" "$repo" 2>&1 && break
+                git clone --depth 1 "https://github.com/paiml/$repo.git" "$repo" 2>&1 && break
                 rm -rf "$repo"  # Clean failed clone before retry
                 echo "::warning::Retry $attempt for $repo clone"
                 sleep 2
@@ -294,7 +310,7 @@
           REPO_NAME: ${{ inputs.repo }}
           RUSTC_WRAPPER: ${{ inputs.enable_sccache && 'sccache' || '' }}
           SCCACHE_DIR: ${{ inputs.enable_sccache && '/sccache' || '' }}
         run: |
           cargo clippy $CLIPPY_ARGS -- -D warnings -A unused-variables 2>&1 || \
           cargo clippy -p "$REPO_NAME" -- -D warnings -A unused-variables 2>&1 || \
           { echo "::error::Clippy failed — check workspace path dependencies"; exit 1; }
@@ -316,7 +332,7 @@
   coverage:
     name: coverage
     if: ${{ !inputs.skip_coverage }}
     runs-on: [self-hosted, clean-room]
     container:
       image: localhost:5000/sovereign-ci:stable@sha256:a7d47ef6e12e23c83075ceff4c41be8f34b00e68639a48bca9e41a2b2c8db80b
       # Phase 3 §5.3 — sccache rustc cache + /var/log/ci-metrics for F9 stats.
@@ -332,7 +348,7 @@
           persist-credentials: false
       - name: Install extra packages
         if: ${{ inputs.extra_pkgs != '' }}
         run: |
           apt-get update -qq && apt-get install -y -qq ${{ inputs.extra_pkgs }} 2>/dev/null || \
           sudo apt-get update -qq && sudo apt-get install -y -qq ${{ inputs.extra_pkgs }} 2>/dev/null || true
       - name: Checkout sibling repos (path deps)
@@ -349,12 +365,20 @@
             [ "$repo" = "${{ inputs.repo }}" ] && continue
             # Remove stale partial clones (file instead of dir)
             if [ -e "$repo" ] && [ ! -d "$repo" ]; then rm -f "$repo"; fi
+            # Validate cached clone. If HEAD missing or fetch fails, nuke + re-clone.
+            # Silent `|| true` on fetch previously masked broken caches (bashrs#197).
             if [ -d "$repo" ]; then
-              # Pull latest to avoid stale cached versions
-              git -C "$repo" fetch --depth 1 origin main 2>/dev/null && git -C "$repo" reset --hard FETCH_HEAD 2>/dev/null || true
-            else
+              if ! git -C "$repo" rev-parse HEAD >/dev/null 2>&1; then
+                echo "::warning::$repo cache invalid (no HEAD) — re-cloning"
+                rm -rf "$repo"
+              elif ! (git -C "$repo" fetch --depth 1 origin main 2>&1 && git -C "$repo" reset --hard FETCH_HEAD >/dev/null 2>&1); then
+                echo "::warning::$repo fetch failed — re-cloning"
+                rm -rf "$repo"
+              fi
+            fi
+            if [ ! -d "$repo" ]; then
               for attempt in 1 2 3; do
-                git clone --depth 1 --quiet "https://github.com/paiml/$repo.git" "$repo" 2>&1 && break
+                git clone --depth 1 "https://github.com/paiml/$repo.git" "$repo" 2>&1 && break
                 rm -rf "$repo"  # Clean failed clone before retry
                 echo "::warning::Retry $attempt for $repo clone"
                 sleep 2
@@ -466,12 +490,20 @@
             [ "$repo" = "${{ inputs.repo }}" ] && continue
             # Remove stale partial clones (file instead of dir)
             if [ -e "$repo" ] && [ ! -d "$repo" ]; then rm -f "$repo"; fi
+            # Validate cached clone. If HEAD missing or fetch fails, nuke + re-clone.
+            # Silent `|| true` on fetch previously masked broken caches (bashrs#197).
             if [ -d "$repo" ]; then
-              # Pull latest to avoid stale cached versions
-              git -C "$repo" fetch --depth 1 origin main 2>/dev/null && git -C "$repo" reset --hard FETCH_HEAD 2>/dev/null || true
-            else
+              if ! git -C "$repo" rev-parse HEAD >/dev/null 2>&1; then
+                echo "::warning::$repo cache invalid (no HEAD) — re-cloning"
+                rm -rf "$repo"
+              elif ! (git -C "$repo" fetch --depth 1 origin main 2>&1 && git -C "$repo" reset --hard FETCH_HEAD >/dev/null 2>&1); then
+                echo "::warning::$repo fetch failed — re-cloning"
+                rm -rf "$repo"
+              fi
+            fi
+            if [ ! -d "$repo" ]; then
               for attempt in 1 2 3; do
-                git clone --depth 1 --quiet "https://github.com/paiml/$repo.git" "$repo" 2>&1 && break
+                git clone --depth 1 "https://github.com/paiml/$repo.git" "$repo" 2>&1 && break
                 rm -rf "$repo"  # Clean failed clone before retry
                 echo "::warning::Retry $attempt for $repo clone"
                 sleep 2
@@ -584,12 +616,20 @@
             [ "$repo" = "${{ inputs.repo }}" ] && continue
             # Remove stale partial clones (file instead of dir)
             if [ -e "$repo" ] && [ ! -d "$repo" ]; then rm -f "$repo"; fi
+            # Validate cached clone. If HEAD missing or fetch fails, nuke + re-clone.
+            # Silent `|| true` on fetch previously masked broken caches (bashrs#197).
             if [ -d "$repo" ]; then
-              # Pull latest to avoid stale cached versions
-              git -C "$repo" fetch --depth 1 origin main 2>/dev/null && git -C "$repo" reset --hard FETCH_HEAD 2>/dev/null || true
-            else
+              if ! git -C "$repo" rev-parse HEAD >/dev/null 2>&1; then
+                echo "::warning::$repo cache invalid (no HEAD) — re-cloning"
+                rm -rf "$repo"
+              elif ! (git -C "$repo" fetch --depth 1 origin main 2>&1 && git -C "$repo" reset --hard FETCH_HEAD >/dev/null 2>&1); then
+                echo "::warning::$repo fetch failed — re-cloning"
+                rm -rf "$repo"
+              fi
+            fi
+            if [ ! -d "$repo" ]; then
               for attempt in 1 2 3; do
-                git clone --depth 1 --quiet "https://github.com/paiml/$repo.git" "$repo" 2>&1 && break
+                git clone --depth 1 "https://github.com/paiml/$repo.git" "$repo" 2>&1 && break
                 rm -rf "$repo"  # Clean failed clone before retry
                 echo "::warning::Retry $attempt for $repo clone"
                 sleep 2