From 8852d199adc49306954e56aa47ef2c5f4122462b Mon Sep 17 00:00:00 2001 From: Jesus Armando Anaya Date: Sun, 9 Aug 2026 22:23:57 -0700 Subject: [PATCH] test(inference): a CI job runs the inference suite with the runtime installed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI installs the base distribution and never the `local-inference` extra, so every test written for the with-runtime side has skipped on every run. The `python` job's absence of the extra is deliberate and stays — it is the half that proves a base install imports none of the runtime and refuses with the install command rather than an `ImportError` — but nothing anywhere ran the other half, which left `tests/architecture/test_optional_runtime.py` asserting a contract that is true by construction wherever torch is not installed, and left a lock bump that broke the adapter, family resolution or the download path free to pass green. A second job on the `format-smoke` mould installs the extra from the lockfile and runs the inference surface on a CPU-only runner, with `VISIONSET_REQUIRE_LOCAL_INFERENCE=1` turning a missing runtime from a skip into an error. Missing-GPU skips stay skips: the runner has no CUDA device and never will, so the half-precision reproduction asks for the runtime and the device as two separate questions, in that order. The predicate five test modules each spelled out now lives once, in `tests/fixtures/local_inference.py`, beside the rule itself — and the rule has a test, written as a `try` rather than as `pytest.raises`, because a skip is a `BaseException` and sails through a `raises` looking for `RuntimeError`: against a build with the guard reverted, `raises` reported the guard's own test as skipped, and a skip is not red. --- .github/workflows/ci.yml | 74 ++++++++++++++++++ CONTRIBUTING.md | 29 ++++++- docs/architecture/backend/inference.md | 9 +++ tests/architecture/test_optional_runtime.py | 6 ++ tests/cli/test_inference_commands.py | 18 +---- tests/fixtures/local_inference.py | 84 +++++++++++++++++++++ tests/inference/test_download_size.py | 8 +- tests/inference/test_fp16.py | 38 +++++----- tests/inference/test_provider.py | 11 ++- tests/inference/test_runtime_gate.py | 74 ++++++++++++++++++ tests/inference/test_weights.py | 8 +- tests/server/test_inference.py | 18 +---- 12 files changed, 311 insertions(+), 66 deletions(-) create mode 100644 tests/fixtures/local_inference.py create mode 100644 tests/inference/test_runtime_gate.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52c99311..ac82d6ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -473,6 +473,80 @@ jobs: cd /tmp /tmp/flow-venv/bin/python "$GITHUB_WORKSPACE/examples/thirty_minute_flow.py" /tmp/flow + # The other half of the inference matrix, on the `format-smoke` mould below: a + # job of its own because the `local-inference` extra is roughly two gigabytes of + # CUDA wheels, which is a cost nobody in the `python` job is paying for. + # + # That job installs the base distribution *deliberately* and is untouched by + # this one. It is the half that proves a machine without the runtime imports + # none of it and refuses with the install command rather than an `ImportError`. + # What it cannot do is exercise the code that only exists when the runtime is + # present — and until this job, nothing did: `tests/architecture/ + # test_optional_runtime.py` asserts that importing the server, the CLI, the job + # registry and `visionset.inference` leaves torch, torchvision, transformers, + # accelerate and huggingface_hub out of `sys.modules`, which is true by + # construction on a machine where none of them is installed. A module-level + # `import torch` under `visionset/inference` — or a lock bump that broke the + # adapter, family resolution or the download path — would have passed green. + # + # `uv sync --locked --extra local-inference`: from the lockfile, never from a + # side index, for the reason the `python` job gives — a resolution happening + # here would be one nobody applied the cool-down to. The locked torch wheels + # carry CUDA and run perfectly well on a runner that has no device. + # + # It runs whole directories, unlike `format-smoke`, and the difference is the + # shadowing that job's comment describes: none of these five distributions ships + # a top-level `tests` package, so this repository's namespace one is intact and + # `tests.fixtures` imports normally here. + inference-smoke: + name: inference smoke (transformers, torch on cpu) + runs-on: ubuntu-latest + env: + # A missing runtime is an error rather than a skip, the + # VISIONSET_REQUIRE_FFMPEG rule: a job that exists to exercise the runtime + # and quietly exercised nothing looks exactly like a passing one. + # + # It says nothing about a missing *GPU*. This runner has no CUDA device and + # never will, so the one test that reproduces the half-precision finding on + # real tensors asks for the runtime and the device as two separate + # questions and skips here on the second — see `tests/inference/test_fp16.py`. + VISIONSET_REQUIRE_LOCAL_INFERENCE: "1" + # Every hub call in these tests is faked, and this is what keeps that true: + # a test that regressed into a real fetch fails here instead of downloading + # gigabytes or hanging on somebody else's uptime. + HF_HUB_OFFLINE: "1" + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Sync environment with the local-inference extra + run: uv sync --locked --extra local-inference + + # The inference surface, named directory by directory rather than left to + # `testpaths`: the rest of the suite already runs in the `python` job, and + # what is wanted here is the code that reads a config, resolves a family, + # drives the hub client and converts tensors — with the real libraries + # loaded rather than a stand-in. + # + # No `-q`: `addopts` in pyproject.toml already carries one, and a second + # suppresses the count line this job exists to put on the record. + - name: The inference surface, with the runtime present + run: | + uv run pytest \ + tests/inference \ + tests/architecture/test_optional_runtime.py \ + tests/server/test_inference.py \ + tests/server/test_suggest.py \ + tests/cli/test_inference_commands.py \ + tests/jobs/test_weights_job.py \ + -rs + # #62's and #63's second acceptance criteria: the tools the exporters exist to # feed actually load what they wrote. Its own job because `ultralytics` brings # torch — roughly two gigabytes — and putting that in front of every `uv sync` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 521d87fe..ea105b28 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -133,7 +133,7 @@ file and read it (`uv run pytest -q > /tmp/out.log 2>&1; echo $?`), or use the s above. The table below is the full list, and it is still wider than the script: the wheel build, -the 30-minute flow, the format smoke tests and the annotator benchmark are left to CI or to +the 30-minute flow, the two smoke suites and the annotator benchmark are left to CI or to a deliberate manual run, because each costs minutes or needs its own install. | Check | Command | In `check.sh` | @@ -153,6 +153,7 @@ a deliberate manual run, because each costs minutes or needs its own install. | Wire action rosters | part of `pnpm test` — `tests/scripts/wire_rosters.test.mjs` compares the two transcriptions of `allowed_actions` (`ui-core`'s `testing/wire.fixtures.ts` and the e2e suite's `_wire.ts`) in both directions. Only the first is typed against the generated union, so the second can drift silently; when it did, the failure surfaced as every gallery spec timing out. It proves the two agree with each other, not that either agrees with the kernel — see #358 | part of `frontend` | | Docs links | part of `pnpm test` — `tests/scripts/docs_links.test.mjs` resolves every internal link and every `#anchor` in all 46 tracked Markdown files (266 links, 551 headings), naming the file, line and dead fragment. External URLs are ignored on purpose: a gate that fails for somebody else's rate limit is one people re-run rather than read. Renaming a heading breaks inbound anchors *silently* — the link just lands at the top of the page — which was a near miss during the `visionset ui` → `visionset server` rename (#329) | part of `frontend` | | Format smoke (ultralytics, pycocotools) | `uv sync --group yolo --group coco && uv run pytest tests/formats/test_*_smoke.py` — their own groups because ultralytics brings torch **and its wheel ships a top-level `tests` package that shadows this repo's**, so run only those files and `uv sync` again afterwards; skips without them, and CI sets `VISIONSET_REQUIRE_ULTRALYTICS=1` / `VISIONSET_REQUIRE_PYCOCOTOOLS=1` so a broken install goes red | — CI | +| Inference smoke (local-inference extra) | `uv sync --extra local-inference` then `VISIONSET_REQUIRE_LOCAL_INFERENCE=1 uv run pytest tests/inference tests/architecture/test_optional_runtime.py tests/server/test_inference.py tests/server/test_suggest.py tests/cli/test_inference_commands.py tests/jobs/test_weights_job.py -rs`, and `uv sync` again afterwards. The **with-runtime** half of the matrix — see [the two halves](#the-two-halves-of-the-inference-matrix) below. Roughly two gigabytes of CUDA wheels, which is why it is opt-in locally; CI's `inference-smoke` job runs it | — CI | | Wheel (build, install, serve) | `bash scripts/build_dist.sh && VISIONSET_REQUIRE_WHEEL=1 uv run pytest tests/packaging` — builds the UI into `_static/`, builds the wheel, installs it in a fresh venv and serves `/app/` from it. Opt-in locally (it costs about a minute); CI's `wheel` job runs it and uploads the artifact | — CI | | The 30-minute flow | `uv run python examples/thirty_minute_flow.py` — the vision document's success metric end to end. CI's `30-minute flow (wheel, end to end)` job runs it from the **installed wheel** in an empty venv, with `ultralytics` required there | — CI | | Version sync | `pnpm version:check` | `generated` | @@ -327,3 +328,29 @@ automation. which turns that skip into a hard failure so a broken install cannot pass unnoticed. The container route needs nothing on the host — `docker/api.Dockerfile` installs it into the image, and CI's `docker` job builds that image and runs the video tests inside it. + +### The two halves of the inference matrix + +`visionset.inference` is tested twice, in two environments, and both halves are deliberate. + +The **without-runtime** half is the ordinary `uv run pytest` and CI's `python` job: no +`local-inference` extra installed. It is what proves a base install is a working install — +that importing the server, the CLI, the job registry and `visionset.inference` pulls in none +of torch, torchvision, transformers, accelerate or huggingface_hub, and that a machine +without them refuses with the install command rather than an `ImportError` from a library the +caller never named. Do not "fix" those skips by installing the extra into the default +environment; they are the test. + +The **with-runtime** half is the table row above and CI's `inference-smoke` job: the extra +installed from `uv.lock`, on a CPU-only runner. It is the only place the lazy-import contract +means anything — on a machine where torch is not installed, "importing the product did not +load torch" is true by construction — and the only place family resolution, capability +derivation, the download path and the tensor conversions meet the real libraries at their +locked versions. + +`VISIONSET_REQUIRE_LOCAL_INFERENCE=1` turns a missing runtime from a skip into an error, so a +broken install goes red instead of quietly shrinking the suite. It says nothing about a +missing **GPU**: no runner has a CUDA device, so the one test that reproduces the +half-precision finding on real tensors asks for the runtime and the device separately and +keeps skipping on the second. Anything that needs a GPU must be written the same way — +`tests/fixtures/local_inference.py` says why. diff --git a/docs/architecture/backend/inference.md b/docs/architecture/backend/inference.md index bf97cf6d..4c970ca3 100644 --- a/docs/architecture/backend/inference.md +++ b/docs/architecture/backend/inference.md @@ -86,6 +86,15 @@ with the optional runtime absent, and [`tests/architecture/test_optional_runtime.py`](../../../tests/architecture/test_optional_runtime.py) proves it in a fresh interpreter. +**That proof needs an environment where the libraries are actually installed.** +On a machine without them, "importing the product left torch out of `sys.modules`" +is true whatever the code does, so the assertion passes and says nothing. CI's +`inference-smoke` job is where it means something: it installs the extra from the +lockfile and runs this file among the rest of the inference surface. The two +halves of that matrix, and the environment variable that keeps the with-runtime +half honest, are described in +[CONTRIBUTING](../../../CONTRIBUTING.md#the-two-halves-of-the-inference-matrix). + The optional dependency group is `local-inference`; `_extra.py` names what it brings and turns a missing one into a sentence carrying the install command. diff --git a/tests/architecture/test_optional_runtime.py b/tests/architecture/test_optional_runtime.py index 2d12b5d5..76151260 100644 --- a/tests/architecture/test_optional_runtime.py +++ b/tests/architecture/test_optional_runtime.py @@ -18,6 +18,12 @@ interpreter has already been filled by the rest of the suite, so the only honest place to ask "what did importing this load?" is an interpreter that has imported nothing else. + +**And an environment where the five are installed, or this proves nothing.** Where +they are absent the assertion holds whatever the code does — a module-level +`import torch` would raise rather than register — so the run that gives this file +its meaning is CI's `inference-smoke` job, which installs the extra from the +lockfile and runs it there. """ from __future__ import annotations diff --git a/tests/cli/test_inference_commands.py b/tests/cli/test_inference_commands.py index ac5c0066..05af8bfa 100644 --- a/tests/cli/test_inference_commands.py +++ b/tests/cli/test_inference_commands.py @@ -7,7 +7,6 @@ from __future__ import annotations -import importlib.util import json from pathlib import Path from typing import Any @@ -15,10 +14,10 @@ import pytest from click.testing import Result from tests.cli._flow import ok, payload, run, runner, workspace +from tests.fixtures.local_inference import without_the_extra from visionset.cli import inference as cli_inference from visionset.cli.main import app -from visionset.inference import MODULES from visionset.inference import weights as weights_module from visionset.inference.integrity import IntegrityReport from visionset.kernel.errors import WeightsDamaged @@ -244,17 +243,6 @@ def test_declining_the_prompt_keeps_the_connection(root: Path) -> None: # --- download ----------------------------------------------------------------- -def _extra_is_installed() -> bool: - """Whether this environment actually carries the local runtime. - - The base development environment does not, and CI's does not either, so the - unstubbed refusal test is the one that runs for real. Guarded rather than - assumed: a contributor who has the extra installed must not see a red suite - for having it. - """ - return all(importlib.util.find_spec(name) is not None for name in MODULES) - - @pytest.fixture() def fetched(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> list[str]: """Record what would have been downloaded, and download nothing. @@ -344,7 +332,7 @@ def test_downloading_an_unknown_connection_exits_one(root: Path, fetched: list[s assert "Error:" in result.stderr -@pytest.mark.skipif(_extra_is_installed(), reason="the local runtime is installed here") +@without_the_extra def test_a_missing_local_runtime_exits_one_with_the_install_command(root: Path) -> None: """Unstubbed. A sentence naming what to run, never a traceback. @@ -441,7 +429,7 @@ def test_the_size_command_prints_the_document_on_json(monkeypatch: pytest.Monkey } -@pytest.mark.skipif(_extra_is_installed(), reason="the local runtime is installed here") +@without_the_extra def test_a_size_without_the_runtime_exits_one_with_the_install_command() -> None: """``size`` opens no workspace, so it carries ``domain_errors`` itself. diff --git a/tests/fixtures/local_inference.py b/tests/fixtures/local_inference.py new file mode 100644 index 00000000..9dc0a4a8 --- /dev/null +++ b/tests/fixtures/local_inference.py @@ -0,0 +1,84 @@ +# usage: from tests.fixtures.local_inference import require_local_inference, without_the_extra +"""Which half of the inference matrix this environment can run, decided once. + +`visionset.inference` is written so that a base install is a working install: it +can create a local connection, list it, and be told exactly what to install +instead of raising an `ImportError` from a library the caller never named. Both +sides of that are tested — the refusals a machine without the runtime gives, and +the code that only exists when the runtime is there — so every such test needs to +know which environment it is in. That predicate lives here rather than in each +module that asks, because five of them used to spell it out and a sixth would +have spelled it slightly differently. + +CI runs the two halves as two jobs. The `python` job installs the base +distribution deliberately, which is what makes the without-runtime tests real +there; `inference-smoke` installs the extra from the lock and sets +`VISIONSET_REQUIRE_LOCAL_INFERENCE=1`. + +**What the variable does is turn a missing runtime from a skip into an error**, +the rule `VISIONSET_REQUIRE_FFMPEG` states for the video suite: a job that exists +to exercise the runtime and quietly exercised nothing looks exactly like a +passing one. It says nothing about a missing *GPU*, which is a separate and +permanent fact about a CI runner — see `require_local_inference`. +""" + +from __future__ import annotations + +import importlib.util +import os +from typing import Final + +import pytest + +from visionset.inference import INSTALL_COMMAND, MODULES + +EXTRA_REQUIRED_ENV: Final = "VISIONSET_REQUIRE_LOCAL_INFERENCE" + +EXTRA_MISSING_HINT: Final = ( + "the local-inference runtime is not installed here. Install it to run VisionSet's " + "with-runtime tests: `uv sync --extra local-inference` in this repository, or " + f"`{INSTALL_COMMAND}` against an installed distribution." +) + +EXTRA_INSTALLED: Final[bool] = all(importlib.util.find_spec(name) is not None for name in MODULES) +"""Whether all five modules of the extra import here. + +`find_spec` rather than an import: this is read at collection, and importing +torch to find out whether torch is installed would put two gigabytes on the +startup path of every test run that does not need it. +""" + +without_the_extra: Final = pytest.mark.skipif( + EXTRA_INSTALLED, reason="the local runtime is installed here" +) +"""For a test whose subject is the *refusal* a base install gives. + +Never an error under `EXTRA_REQUIRED_ENV`, and that is the whole asymmetry: +these tests are the `python` job's, and their skipping in `inference-smoke` is +the job doing its job. A contributor who installs the extra must not get a red +suite for having it either. +""" + + +def require_local_inference() -> None: + """Skip locally, fail where the runtime was supposed to be installed. + + `tests.fixtures.media.require_ffmpeg`, called from inside a test rather than + at module level: the modules holding these tests hold the without-runtime + half too, and a module-level skip would take the tests the `python` job + exists to run along with them. + + **Missing runtime, not missing GPU.** A CI runner has no CUDA device and + never will, so a test that needs one keeps skipping honestly under this + variable — it asks for the runtime through this function and for the device + separately, afterwards. A single condition covering both would make the job + permanently red for the one reason nobody can fix. + """ + if EXTRA_INSTALLED: + return + if os.environ.get(EXTRA_REQUIRED_ENV) == "1": + raise RuntimeError( + f"{EXTRA_MISSING_HINT} " + f"({EXTRA_REQUIRED_ENV}=1 is set, so a missing runtime is an error, not a skip.)" + ) + pytest.skip(EXTRA_MISSING_HINT) diff --git a/tests/inference/test_download_size.py b/tests/inference/test_download_size.py index 5f2fde25..5587f6a6 100644 --- a/tests/inference/test_download_size.py +++ b/tests/inference/test_download_size.py @@ -12,18 +12,16 @@ from __future__ import annotations -import importlib.util from dataclasses import dataclass from typing import Any import pytest +from tests.fixtures.local_inference import without_the_extra -from visionset.inference import MODULES, DownloadSizes, download_size, known_sizes, measure +from visionset.inference import DownloadSizes, download_size, known_sizes, measure from visionset.inference import weights as weights_module from visionset.kernel.errors import LocalInferenceUnavailable -EXTRA_INSTALLED = all(importlib.util.find_spec(name) is not None for name in MODULES) - @dataclass(frozen=True) class FakeSibling: @@ -177,7 +175,7 @@ def model_info(*_: object, **__: object) -> FakeInfo: assert "Repository Not Found" in str(raised.value) -@pytest.mark.skipif(EXTRA_INSTALLED, reason="the local runtime is installed here") +@without_the_extra def test_a_missing_hub_client_names_the_install_command() -> None: """Unstubbed: the size is read with the client that would do the fetching, so a machine without the extra is refused here too — with the remedy.""" diff --git a/tests/inference/test_fp16.py b/tests/inference/test_fp16.py index 247a6d43..d66fc761 100644 --- a/tests/inference/test_fp16.py +++ b/tests/inference/test_fp16.py @@ -13,17 +13,18 @@ everywhere, including on a base install with no torch at all, and it is what catches the shim being dropped. - **The reproduction**, driven with real half-precision CUDA tensors through the - real `torch.nn.functional.grid_sample`. It is skipped without a GPU — so it is - skipped in CI — and it is the one that would have caught the finding in the - first place. + real `torch.nn.functional.grid_sample`. It needs the runtime *and* a GPU, which + it asks for as two separate questions; CI has the first and none of its runners + has the second, so it is skipped there. It is the one that would have caught + the finding in the first place. """ from __future__ import annotations -import importlib.util from typing import Any import pytest +from tests.fixtures.local_inference import require_local_inference from visionset.inference._fp16 import ( forward_guard, @@ -216,17 +217,6 @@ def test_everything_else_is_full_precision(spelling: str | None) -> None: # --- the reproduction, on real hardware --------------------------------------- -def _cuda_is_available() -> bool: - if importlib.util.find_spec("torch") is None: - return False - import torch as real_torch - - return bool(real_torch.cuda.is_available()) - - -@pytest.mark.skipif( - not _cuda_is_available(), reason="needs the local-inference extra and a CUDA device" -) def test_a_half_precision_grid_sample_survives_a_float32_grid_on_cuda() -> None: """The measured failure, reproduced and then fixed, with real tensors. @@ -236,13 +226,23 @@ def test_a_half_precision_grid_sample_survives_a_float32_grid_on_cuda() -> None: found Float`, which is exactly what the spike hit inside deformable attention; inside `forward_guard` it returns. - Skipped without a GPU, so it does not run in CI. It is here because the - stubbed tests above prove the *rule* and only this one proves the rule was - the right one — and because the next person to touch the shim will want a way - to check it on a machine that has the hardware. + It is here because the stubbed tests above prove the *rule* and only this one + proves the rule was the right one — and because the next person to touch the + shim will want a way to check it on a machine that has the hardware. + + **Two gates, asked in this order, and the order is the point.** The runtime is + required first, so a build that was meant to carry torch and does not is an + error. The device is asked for second, and a missing one stays a skip forever: + no CI runner here has a GPU, and folding the two into one condition would make + the `inference-smoke` job permanently red for the only reason nobody can fix. """ + require_local_inference() + import torch as real_torch + if not real_torch.cuda.is_available(): + pytest.skip("needs a CUDA device") + values = real_torch.randn(1, 1, 8, 8, device="cuda", dtype=real_torch.float16) grid = real_torch.zeros(1, 4, 4, 2, device="cuda", dtype=real_torch.float32) diff --git a/tests/inference/test_provider.py b/tests/inference/test_provider.py index b1866e71..932e9425 100644 --- a/tests/inference/test_provider.py +++ b/tests/inference/test_provider.py @@ -18,13 +18,13 @@ from __future__ import annotations -import importlib.util from pathlib import Path from uuid import uuid4 import pytest +from tests.fixtures.local_inference import require_local_inference, without_the_extra -from visionset.inference import MODULES, LocalTransformersProvider, provider_for +from visionset.inference import LocalTransformersProvider, provider_for from visionset.inference import providers as providers_module from visionset.inference.nms import DEFAULT_IOU_THRESHOLD from visionset.inference.transformers_provider import prompt_text, regions_from @@ -43,8 +43,6 @@ ) from visionset.kernel.ports import ModelProvider -EXTRA_INSTALLED = all(importlib.util.find_spec(name) is not None for name in MODULES) - def local( setup_state: ConnectionSetupState = ConnectionSetupState.NOT_SET_UP, @@ -63,7 +61,7 @@ def local( # --- resolving a connection to something that can answer ---------------------- -@pytest.mark.skipif(EXTRA_INSTALLED, reason="the local runtime is installed here") +@without_the_extra def test_a_ready_connection_without_the_runtime_names_the_install_command( tmp_path: Path, ) -> None: @@ -78,7 +76,6 @@ def test_a_ready_connection_without_the_runtime_names_the_install_command( assert 'pip install "visionset[local-inference]"' in str(raised.value) -@pytest.mark.skipif(not EXTRA_INSTALLED, reason="needs the local-inference extra") def test_a_ready_local_connection_resolves_to_a_provider( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -93,6 +90,8 @@ def test_a_ready_local_connection_resolves_to_a_provider( to a fallback, so a real config is now the difference between this test building a provider and it exercising the refusal that has its own test. """ + require_local_inference() + monkeypatch.setattr(providers_module, "family_of", lambda *_, **__: "grounding-dino") built = provider_for(local(ConnectionSetupState.READY), workspace_root=tmp_path) assert isinstance(built, ModelProvider) diff --git a/tests/inference/test_runtime_gate.py b/tests/inference/test_runtime_gate.py new file mode 100644 index 00000000..e31157c8 --- /dev/null +++ b/tests/inference/test_runtime_gate.py @@ -0,0 +1,74 @@ +"""The gate that decides which half of the inference matrix runs, tested itself. + +`require_local_inference` is the only reason CI's `inference-smoke` job means +anything: it is what turns a broken install of the optional runtime from a +quietly shrinking suite into a red build. A rule nothing exercises is a comment, +and this one is invisible when it works — the whole job passes either way — so it +gets a test that drives all three of its answers directly. + +Nothing here needs the runtime installed, or absent: the predicate is patched, so +the same three assertions run identically in both CI jobs and on any developer +machine. +""" + +from __future__ import annotations + +import pytest +from tests.fixtures import local_inference +from tests.fixtures.local_inference import EXTRA_REQUIRED_ENV, require_local_inference + + +def test_a_missing_runtime_is_a_skip_where_nothing_asked_for_it( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A base install is a working install, and its test run says so by skipping. + + The default has to be the quiet one: most machines that run this suite have + no reason to carry two gigabytes of CUDA wheels, and a red suite for not + having them would be a demand rather than a check. + """ + monkeypatch.setattr(local_inference, "EXTRA_INSTALLED", False) + monkeypatch.delenv(EXTRA_REQUIRED_ENV, raising=False) + + with pytest.raises(pytest.skip.Exception) as raised: + require_local_inference() + assert "local-inference" in str(raised.value) + + +def test_a_missing_runtime_is_an_error_where_the_job_installed_it( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The flip, and the whole point of the variable. + + Reverting the skip-to-error branch in `require_local_inference` turns this + test red — which is the only thing standing between `inference-smoke` and the + failure it exists to prevent: an install that broke, a suite that skipped + every test needing the runtime, and a green tick reporting it. + + **Written as a `try` rather than as `pytest.raises(RuntimeError)`, and that is + the whole reason it works.** A skip is raised as a `BaseException`, which + sails straight through a `raises` looking for `RuntimeError` and skips *this* + test — so against a build where the flip had been reverted, the guard's own + test reported "skipped", not "failed", and a skip is not red. Catching the + skip by name is what turns the regression into a failure somebody sees. + """ + monkeypatch.setattr(local_inference, "EXTRA_INSTALLED", False) + monkeypatch.setenv(EXTRA_REQUIRED_ENV, "1") + + try: + require_local_inference() + except RuntimeError as error: + assert EXTRA_REQUIRED_ENV in str(error) + except pytest.skip.Exception as skipped: + pytest.fail(f"a missing runtime skipped under {EXTRA_REQUIRED_ENV}=1: {skipped}") + else: + pytest.fail(f"a missing runtime did nothing under {EXTRA_REQUIRED_ENV}=1") + + +def test_a_present_runtime_asks_nothing_of_either(monkeypatch: pytest.MonkeyPatch) -> None: + """With the runtime there, the variable changes nothing — it only ever + describes what to do about its absence.""" + monkeypatch.setattr(local_inference, "EXTRA_INSTALLED", True) + monkeypatch.setenv(EXTRA_REQUIRED_ENV, "1") + + assert require_local_inference() is None diff --git a/tests/inference/test_weights.py b/tests/inference/test_weights.py index fa68e94e..cebb088b 100644 --- a/tests/inference/test_weights.py +++ b/tests/inference/test_weights.py @@ -13,15 +13,15 @@ from __future__ import annotations -import importlib.util from collections.abc import Iterator from pathlib import Path from typing import Any from uuid import uuid4 import pytest +from tests.fixtures.local_inference import without_the_extra -from visionset.inference import MODULES, cache_root, fetch_weights, with_families +from visionset.inference import cache_root, fetch_weights, with_families from visionset.inference import weights as weights_module from visionset.inference.weights import MODELS_DIRNAME, download from visionset.kernel.domain import ( @@ -36,8 +36,6 @@ ) from visionset.kernel.services import InferenceConnectionService, WorkspaceService -EXTRA_INSTALLED = all(importlib.util.find_spec(name) is not None for name in MODULES) - @pytest.fixture() def workspace(tmp_path: Path) -> Iterator[WorkspaceService]: @@ -357,7 +355,7 @@ def test_downloading_for_an_http_connection_refuses_rather_than_fetching_nothing download(remote, into=Path("/nowhere")) -@pytest.mark.skipif(EXTRA_INSTALLED, reason="the local runtime is installed here") +@without_the_extra def test_a_missing_hub_client_names_the_install_command(tmp_path: Path) -> None: """The translation, unstubbed: an `ImportError` becomes a kernel error whose message is the remedy.""" diff --git a/tests/server/test_inference.py b/tests/server/test_inference.py index c9265a00..c3cd017b 100644 --- a/tests/server/test_inference.py +++ b/tests/server/test_inference.py @@ -5,7 +5,6 @@ is `tests/kernel/test_inference_connections.py`'s subject. """ -import importlib.util from collections.abc import Iterator from pathlib import Path from typing import Any @@ -13,10 +12,10 @@ import pytest from fastapi.testclient import TestClient +from tests.fixtures.local_inference import without_the_extra from tests.server._api import api_client from tests.server._jobs import InlineDispatcher, ManualDispatcher -from visionset.inference import MODULES from visionset.inference import weights as weights_module from visionset.inference.integrity import IntegrityReport from visionset.jobs import integrity as job_module @@ -310,17 +309,6 @@ def test_deleting_an_unknown_connection_is_not_found(client: TestClient) -> None # --- downloading weights ------------------------------------------------------ -def _extra_is_installed() -> bool: - """Whether this environment actually carries the local runtime. - - The base development environment does not, and CI's does not either, so the - unstubbed refusal test below is the one that runs for real. Guarded rather - than assumed, because a contributor with the extra installed must not see a - red suite for having it. - """ - return all(importlib.util.find_spec(name) is not None for name in MODULES) - - def test_downloading_answers_202_and_points_at_its_job( tmp_path: Path, runtime_present: None ) -> None: @@ -457,7 +445,7 @@ def test_downloading_an_unknown_connection_is_not_found( assert response.json()["code"] == "INFERENCE_CONNECTION_NOT_FOUND" -@pytest.mark.skipif(_extra_is_installed(), reason="the local runtime is installed here") +@without_the_extra def test_a_missing_local_runtime_refuses_with_the_install_command(client: TestClient) -> None: """Unstubbed, and the message is the remedy. @@ -702,7 +690,7 @@ def test_the_size_route_wants_both_halves_of_the_pair(client: TestClient) -> Non assert response.status_code == 422, response.text -@pytest.mark.skipif(_extra_is_installed(), reason="the local runtime is installed here") +@without_the_extra def test_a_size_without_the_runtime_carries_the_install_command(client: TestClient) -> None: """Unstubbed, and the same refusal the download gives.