mudler · richiejp · Jul 21, 2026 · Jul 21, 2026 · Jul 21, 2026 · Jul 21, 2026
diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md
@@ -21,6 +21,9 @@ Let's say the user wants to build a particular backend for a given platform. For
 The core Go suites (`./pkg`, `./core`, plus the in-process integration suite `./tests/e2e`) are covered by a **strict, monotonic coverage ratchet**:
 
 - `make test-coverage` — runs the suites with `covermode=atomic` instrumentation and writes a merged profile to `coverage/coverage.out`. Uses the same prerequisites as `make test`.
+  - Prints per-root wall time and the slowest specs/hooks exceeding `COVERAGE_SLOW_SPEC_THRESHOLD` (default 3 seconds, capped by `COVERAGE_SLOW_SPEC_LIMIT`, default 25 per root); machine-readable root timings are written to `coverage/timings.tsv`.
+  - Verbose Ginkgo output is written to `coverage/logs/<root>.log`, with the prior run retained as `<root>.log.previous`. The terminal prints one status line per root and a short failure extract. If any suite fails, no merged profile is produced and the percentage ratchet is explicitly not run. A lock under `coverage/` rejects concurrent runs, which would otherwise corrupt their shared profiles and logs.
+  - Suites run in parallel by default and each recursive root invocation has a five-minute budget. Override auto-detected parallelism with `COVERAGE_PROCS`; tune diagnostics with `COVERAGE_SUITE_TIMEOUT` and `COVERAGE_PROGRESS_AFTER`. A timeout is a performance failure to investigate, not a reason to raise the committed default.
   - **`--coverpkg` (`COVERAGE_COVERPKG = core/...,pkg/...`):** coverage is attributed to the core+pkg packages, not just the package under test. This is what lets the in-process `tests/e2e` suite (which drives the real HTTP server over loopback via `application.New`) credit the `core/http/endpoints/...` handlers it exercises — folding it in roughly doubled endpoint coverage (e.g. `endpoints/openai` 13.6% → 52%). The denominator is therefore *all* of `core`+`pkg` (minus generated proto, dropped via `COVERAGE_EXCLUDE_RE`), so the number isn't comparable to a plain per-package figure.
   - **Integration suites (`COVERAGE_E2E_ROOTS = ./tests/e2e`)** run non-recursively (excludes `tests/e2e/distributed`, which needs containers) with `--label-filter=!real-models` (those need a downloaded model) against the mock backend built by `prepare-test`. `tests/integration` is deliberately excluded — it needs `make backends/local-store`, which the coverage CI job doesn't build.
   - **Flake note:** folding integration tests into a *strict* gate means a hard e2e failure (or a spec that silently stops running) can fail the coverage gate, not just the test. `--flake-attempts` absorbs transient retryable failures; covermode=atomic keeps line coverage deterministic otherwise.

diff --git a/.github/workflows/external-probes.yml b/.github/workflows/external-probes.yml
@@ -0,0 +1,38 @@
+---
+name: external compatibility probes
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '23 4 * * 1'
+
+permissions:
+  contents: read
+
+jobs:
+  external-probe-huggingface-xet:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.26.x'
+          cache: false
+      - name: Probe Hugging Face Xet compatibility
+        run: LOCALAI_HF_XET_SMOKE=1 go test ./pkg/huggingface-api -ginkgo.focus='pinned public Xet fixture' -count=1
+
+  external-probe-sigstore:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.26.x'
+          cache: false
+      - name: Probe public Sigstore compatibility
+        env:
+          LOCALAI_COSIGN_LIVE: '1'
+          LOCALAI_COSIGN_LIVE_IMAGE: ${{ vars.LOCALAI_COSIGN_LIVE_IMAGE }}
+          LOCALAI_COSIGN_LIVE_ISSUER: ${{ vars.LOCALAI_COSIGN_LIVE_ISSUER }}
+          LOCALAI_COSIGN_LIVE_IDENTITY_REGEX: ${{ vars.LOCALAI_COSIGN_LIVE_IDENTITY_REGEX }}
+        run: go test ./pkg/oci/cosignverify -ginkgo.focus='VerifyImage' -count=1
diff --git a/.github/workflows/test-resource-refresh.yml b/.github/workflows/test-resource-refresh.yml
@@ -0,0 +1,76 @@
+---
+name: refresh offline test resources
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '17 3 * * 1'
+
+permissions:
+  contents: read
+  issues: write
+  packages: write
+
+jobs:
+  refresh:
+    strategy:
+      fail-fast: false
+      matrix:
+        resource-set: [default, distributed-e2e, aio]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.26.x'
+          cache: true
+      - uses: oras-project/setup-oras@v1
+      - name: Verify upstream resources and build compressed cache
+        id: refresh
+        continue-on-error: true
+        env:
+          LOCALAI_TEST_RESOURCES_ONLINE: '1'
+        run: |
+          set -o pipefail
+          make update-offline-test-cache TEST_RESOURCE_SET=${{ matrix.resource-set }} 2>&1 | tee resource-refresh.log
+      - name: Upload investigation evidence
+        if: steps.refresh.outcome == 'failure'
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-resource-investigation-${{ matrix.resource-set }}-${{ github.run_id }}
+          path: resource-refresh.log
+      - name: Open or update investigation issue
+        if: steps.refresh.outcome == 'failure'
+        env:
+          GH_TOKEN: ${{ secrets.LOCALAI_BOT_TOKEN || github.token }}
+          RESOURCE_SET: ${{ matrix.resource-set }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          title="test resource integrity investigation: ${RESOURCE_SET}"
+          body=$(printf '%s\n\n%s\n' \
+            "The scheduled offline-resource refresh failed for \`${RESOURCE_SET}\`." \
+            "Do not update the manifest digest blindly. Download the evidence artifact from ${RUN_URL}, compare upstream checksums/signatures and release notes, inspect redirects, and search the [GitHub Advisory Database](https://github.com/advisories) and [OSV](https://osv.dev). Retry from a declared mirror to distinguish source drift from corruption.")
+          existing=$(gh issue list --state open --search "${title} in:title" --json number --jq '.[0].number // empty')
+          if [ -n "$existing" ]; then
+            gh issue comment "$existing" --body "$body"
+          else
+            gh issue create --title "$title" --body "$body"
+          fi
+      - name: Log in to GHCR
+        if: steps.refresh.outcome == 'success'
+        run: echo "${{ github.token }}" | oras login ghcr.io -u "${{ github.actor }}" --password-stdin
+      - name: Publish compressed cache as an OCI artifact
+        if: steps.refresh.outcome == 'success'
+        env:
+          RESOURCE_SET: ${{ matrix.resource-set }}
+        run: |
+          repository=$(printf '%s' "ghcr.io/${GITHUB_REPOSITORY}/localai-test-resources" | tr '[:upper:]' '[:lower:]')
+          digest=$(jq -r --arg set "$RESOURCE_SET" '.bundles[$set] | sub("sha256:"; "sha256-")' test-resources/manifests/lock.json)
+          oras push \
+            --artifact-type application/vnd.localai.test-resources.v1 \
+            "${repository}:${RESOURCE_SET},${RESOURCE_SET}-${digest}" \
+            ".cache/test-resources/bundles/${RESOURCE_SET}.tar.zst:application/vnd.localai.test-resources.bundle.v1+zstd" \
+            "test-resources/manifests/${RESOURCE_SET}.json:application/vnd.localai.test-resources.manifest.v1+json"
+      - name: Fail after preserving evidence
+        if: steps.refresh.outcome == 'failure'
+        run: exit 1
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -39,6 +39,8 @@ jobs:
       # You can test your matrix by printing the current Go version
       - name: Display Go version
         run: go version
+      - name: Download Go modules
+        run: go mod download
       - name: Proto Dependencies
         run: |
           # Install protoc
@@ -58,13 +60,30 @@ jobs:
           node-version: '22'
       - name: Build React UI
         run: make react-ui
+      - name: Record and pack declared test resources
+        run: LOCALAI_TEST_RESOURCES_ONLINE=1 make update-offline-test-cache TEST_RESOURCE_SET=default
+      - name: Transfer local test-resource bundle
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-resources-default-${{ github.run_id }}
+          include-hidden-files: true
+          path: |
+            .cache/test-resources/bundles/default.tar.zst
+            test-resources/manifests/lock.json
+      - name: Clear recorded resource cache
+        run: rm -rf .cache/test-resources
+      - name: Restore local test-resource bundle
+        uses: actions/download-artifact@v4
+        with:
+          name: test-resources-default-${{ github.run_id }}
+          path: .
       # Runs the core suite with coverage and fails if total coverage dropped
       # below the committed baseline (coverage-baseline.txt). The gate is
       # strict — any decrease fails. Raise the baseline with
       # `make test-coverage-baseline` and commit it when coverage rises.
       - name: Test (with coverage gate)
         run: |
-          PATH="$PATH:/root/go/bin" make --jobs 5 --output-sync=target test-coverage-check
+          LOCALAI_TEST_KERNEL_ENFORCE=1 PATH="$PATH:/root/go/bin" make --jobs 5 --output-sync=target test-coverage-check
       - name: Upload coverage report
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
@@ -100,6 +119,8 @@ jobs:
       # You can test your matrix by printing the current Go version
       - name: Display Go version
         run: go version
+      - name: Download Go modules
+        run: go mod download
       - name: Dependencies
         run: |
           brew install protobuf grpc make protoc-gen-go protoc-gen-go-grpc libomp llvm opus ffmpeg
@@ -118,7 +139,7 @@ jobs:
           # Used to run the newer GNUMake version from brew that supports --output-sync
           export PATH="/opt/homebrew/opt/make/libexec/gnubin:$PATH"
           PATH="$PATH:$HOME/go/bin" make protogen-go
-          PATH="$PATH:$HOME/go/bin" BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DGGML_F16C=OFF -DGGML_AVX512=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF" make --jobs 4 --output-sync=target test
+          PATH="$PATH:$HOME/go/bin" BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DGGML_F16C=OFF -DGGML_AVX512=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF" make --jobs 4 --output-sync=target TEST_RESOURCE_SET=default-darwin test
       - name: Setup tmate session if tests fail
         if: ${{ failure() }}
         uses: mxschmitt/action-tmate@v3.23

diff --git a/.github/workflows/tests-aio.yml b/.github/workflows/tests-aio.yml
@@ -76,7 +76,9 @@ jobs:
           PATH="$PATH:$HOME/go/bin" make protogen-go
       - name: Test
         run: |
-            PATH="$PATH:$HOME/go/bin" make backends/local-store backends/silero-vad backends/llama-cpp backends/whisper backends/piper backends/stablediffusion-ggml docker-build-e2e e2e-aio
+            PATH="$PATH:$HOME/go/bin" make backends/local-store backends/silero-vad backends/llama-cpp backends/whisper backends/piper backends/stablediffusion-ggml docker-build-e2e
+            LOCALAI_TEST_RESOURCES_ONLINE=1 PATH="$PATH:$HOME/go/bin" make update-offline-test-cache TEST_RESOURCE_SET=aio
+            LOCALAI_BACKEND_DIR="$GITHUB_WORKSPACE/backends" LOCALAI_MODELS_DIR="$GITHUB_WORKSPACE/tests/e2e-aio/models" LOCALAI_IMAGE_TAG=tests LOCALAI_IMAGE=local-ai PATH="$PATH:$HOME/go/bin" make run-e2e-aio
       - name: Setup tmate session if tests fail
         if: ${{ failure() }}
         uses: mxschmitt/action-tmate@v3.23

diff --git a/.github/workflows/tests-e2e.yml b/.github/workflows/tests-e2e.yml
@@ -60,9 +60,12 @@ jobs:
           node-version: '22'
       - name: Build React UI
         run: make react-ui
+      - name: Record declared distributed test resources
+        run: LOCALAI_TEST_RESOURCES_ONLINE=1 make update-offline-test-cache TEST_RESOURCE_SET=distributed-e2e
       - name: Test Backend E2E
         run: |
           PATH="$PATH:$HOME/go/bin" make build-mock-backend test-e2e
+          PATH="$PATH:$HOME/go/bin" make test-e2e-distributed
       - name: Setup tmate session if tests fail
         if: ${{ failure() }}
         uses: mxschmitt/action-tmate@v3.23

diff --git a/Makefile b/Makefile
@@ -99,11 +99,20 @@ COVERAGE_COVERPKG?=github.com/mudler/LocalAI/core/...,github.com/mudler/LocalAI/
 ## the coverage CI job doesn't do.
 COVERAGE_E2E_ROOTS?=./tests/e2e
 COVERAGE_E2E_LABELS?=!real-models
+COVERAGE_PROCS?=0
+COVERAGE_SUITE_TIMEOUT?=5m
+COVERAGE_PROGRESS_AFTER?=30s
+COVERAGE_SLOW_SPEC_THRESHOLD?=3
+COVERAGE_SLOW_SPEC_LIMIT?=25
 ## Drop generated protobuf from the denominator (it has no tests by design).
 COVERAGE_EXCLUDE_RE?=grpc/proto/.*[.]pb[.]go
+TEST_RESOURCE_SET?=default
+TEST_RESOURCE_CACHE?=$(abspath ./.cache/test-resources)
+TEST_RESOURCE_MANIFESTS?=$(abspath ./test-resources/manifests)
+OFFLINE_RUN=$(abspath ./scripts/run-test-offline.sh)
 
 
-.PHONY: all test test-coverage test-coverage-baseline test-coverage-check test-backend-cpp test-build-scripts test-ui test-ui-coverage-baseline test-ui-coverage-check build vendor lint lint-all
+.PHONY: all test prepare-offline-test-cache update-offline-test-cache test-coverage test-coverage-baseline test-coverage-check test-backend-cpp test-build-scripts test-ui test-ui-coverage-baseline test-ui-coverage-check install-hooks build vendor lint lint-all
 
 all: help
 
@@ -204,11 +213,21 @@ prepare-test: protogen-go build-mock-backend
 ## now drives the mock-backend binary built by build-mock-backend; real-backend
 ## inference moved into tests/e2e-backends/ (per-backend, path-filtered) and
 ## tests/e2e-aio/ (nightly).
+prepare-offline-test-cache:
+	@test -n "$(TEST_RESOURCE_SET)" || { echo 'TEST_RESOURCE_SET is required, for example: make prepare-offline-test-cache TEST_RESOURCE_SET=default'; exit 2; }
+	$(GOCMD) run ./cmd/test-resources prepare "$(TEST_RESOURCE_SET)" "$(TEST_RESOURCE_MANIFESTS)" "$(TEST_RESOURCE_CACHE)"
+
+update-offline-test-cache:
+	@test -n "$(TEST_RESOURCE_SET)" || { echo 'TEST_RESOURCE_SET is required, for example: make update-offline-test-cache TEST_RESOURCE_SET=default'; exit 2; }
+	@test "$$LOCALAI_TEST_RESOURCES_ONLINE" = 1 || { echo 'Set LOCALAI_TEST_RESOURCES_ONLINE=1 to enter explicit online record mode'; exit 2; }
+	$(GOCMD) run ./cmd/test-resources update "$(TEST_RESOURCE_SET)" "$(TEST_RESOURCE_MANIFESTS)" "$(TEST_RESOURCE_CACHE)"
+
+test: TEST_RESOURCE_SET=default
 test: prepare-test
 	@echo 'Running tests'
 	export GO_TAGS="debug"
 	OPUS_SHIM_LIBRARY=$(abspath ./pkg/opus/shim/libopusshim.so) \
-	$(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) --fail-fast -v -r $(TEST_PATHS)
+	$(OFFLINE_RUN) $(TEST_RESOURCE_SET) $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) --fail-fast -v -r $(TEST_PATHS)
 
 ## Compiles and runs the standalone C++ unit tests for the backends (pure
 ## helpers that depend only on the stdlib + nlohmann/json, no full backend
@@ -243,15 +262,21 @@ test-python-helpers:
 ## and writes a merged profile to $(COVERAGE_PROFILE). Deliberately omits
 ## --fail-fast so a single failure doesn't truncate the coverage number, and
 ## uses covermode=atomic so the result is deterministic. Prints the total.
+test-coverage: TEST_RESOURCE_SET=default
 test-coverage: prepare-test
-	@echo 'Running tests with coverage'
+	@echo 'Running tests with coverage (test failures stop before the percentage ratchet)'
 	GINKGO_TAGS="$(COVERAGE_TAGS)" \
 	COVERAGE_COVERPKG="$(COVERAGE_COVERPKG)" \
 	COVERAGE_E2E_ROOTS="$(COVERAGE_E2E_ROOTS)" \
 	COVERAGE_E2E_LABELS="$(COVERAGE_E2E_LABELS)" \
+	COVERAGE_PROCS="$(COVERAGE_PROCS)" \
+	COVERAGE_SUITE_TIMEOUT="$(COVERAGE_SUITE_TIMEOUT)" \
+	COVERAGE_PROGRESS_AFTER="$(COVERAGE_PROGRESS_AFTER)" \
+	COVERAGE_SLOW_SPEC_THRESHOLD="$(COVERAGE_SLOW_SPEC_THRESHOLD)" \
+	COVERAGE_SLOW_SPEC_LIMIT="$(COVERAGE_SLOW_SPEC_LIMIT)" \
 	COVERAGE_EXCLUDE_RE='$(COVERAGE_EXCLUDE_RE)' \
 	OPUS_SHIM_LIBRARY=$(abspath ./pkg/opus/shim/libopusshim.so) \
-	scripts/run-coverage.sh $(COVERAGE_DIR) $(COVERAGE_PROFILE) $(TEST_FLAKES) $(COVERAGE_ROOTS)
+	$(OFFLINE_RUN) $(TEST_RESOURCE_SET) scripts/run-coverage.sh $(COVERAGE_DIR) $(COVERAGE_PROFILE) $(TEST_FLAKES) $(COVERAGE_ROOTS)
 	@$(GOCMD) tool cover -html=$(COVERAGE_PROFILE) -o $(COVERAGE_DIR)/coverage.html
 	@$(GOCMD) tool cover -func=$(COVERAGE_PROFILE) | tail -n1
 
@@ -267,6 +292,7 @@ test-coverage-baseline: test-coverage
 ## run-to-run jitter from the in-process tests/e2e suite folded in via
 ## --coverpkg (timing-dependent which handler lines execute).
 test-coverage-check: test-coverage
+	@echo 'Running coverage percentage ratchet'
 	@scripts/coverage-check.sh $(COVERAGE_PROFILE) $(COVERAGE_BASELINE)
 
 ########################################################
@@ -331,16 +357,18 @@ e2e-aio:
 	LOCALAI_IMAGE=local-ai \
 	$(MAKE) run-e2e-aio
 
+run-e2e-aio: TEST_RESOURCE_SET=aio
 run-e2e-aio: protogen-go
 	@echo 'Running e2e AIO tests'
-	$(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e-aio
+	$(OFFLINE_RUN) $(TEST_RESOURCE_SET) $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e-aio
 
 # Distributed architecture e2e (PostgreSQL + NATS via testcontainers).
 # Includes NatsJWT specs (JWT-enabled NATS). Requires Docker.
 # VLLMMultinode is excluded here; use test-e2e-vllm-multinode for that.
+test-e2e-distributed: TEST_RESOURCE_SET=distributed-e2e
 test-e2e-distributed: protogen-go
 	@echo 'Running distributed e2e tests (label Distributed, incl. NatsJWT)'
-	$(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode' --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e/distributed
+	$(OFFLINE_RUN) $(TEST_RESOURCE_SET) $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode' --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e/distributed
 
 # vLLM multi-node DP smoke (CPU). Builds local-ai:tests and the
 # cpu-vllm backend from the current working tree, then drives a
@@ -382,7 +410,7 @@ test-e2e: build-mock-backend build-cloud-proxy-backend prepare-e2e run-e2e-image
 	@echo 'Running e2e tests'
 	BUILD_TYPE=$(BUILD_TYPE) \
 	LOCALAI_API=http://$(E2E_BRIDGE_IP):5390 \
-	$(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e
+	$(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='!Distributed' --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e
 	$(MAKE) clean-mock-backend
 	$(MAKE) clean-cloud-proxy-backend
 	$(MAKE) teardown-e2e