test: add docs() tests

.github/workflows/consumer_test.yml → .github/workflows/downstream_compatibility.yml

@@ -11,7 +11,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # *******************************************************************************
 
-name: Consumer Tests
+name: Downstream Compatibility
 on:
   workflow_call:
   pull_request:
@@ -48,7 +48,7 @@ jobs:
         run: bazel run //:ide_support
 
       - name: Run Consumer tests
-        run: .venv_docs/bin/python -m pytest -vv -s src/tests/test_consumer.py -k "$CONSUMER"
+        run: .venv_docs/bin/python -m pytest -vv -s src/tests/downstream_compatibility -k "$CONSUMER"
         env:
           FORCE_COLOR: "1"
           TERM: xterm-256color

.github/workflows/test.yml

@@ -25,6 +25,7 @@ jobs:
         uses: step-security/harden-runner@v2.18.0
         with:
           egress-policy: audit
+
       - name: Checkout repository (Handle all events)
         uses: actions/checkout@v4.2.2
         with:
@@ -38,23 +39,32 @@ jobs:
           repository-cache: true
           bazelisk-cache: true
           cache-save: ${{ github.event_name == 'push' }}
-      - name: Run test targets
-        run: |
-          bazel run --lockfile_mode=error //:ide_support
-          bazel test --lockfile_mode=error //...
 
-      - name: Prepare bundled consumer report
-        if: always()
+      - name: Setup venv
+        run: bazel run --lockfile_mode=error //:ide_support
+
+      - name: Run docs_bzl tests
+        run: .venv_docs/bin/python -m pytest -vv src/tests/docs_bzl
+
+      - name: Run bazel test targets
+        run: bazel test --lockfile_mode=error //... --build_tests_only
+
+      - name: Build all targets
+        # Skip tests, since some are non buildable negative tests
+        run: bazel build --lockfile_mode=error //... -- -//src/tests/...
+
+      - name: Prepare bundled test report
+        if: ! cancelled()
         # Creating tests-report directory
         # Follow Symlinks via '-L' to copy correctly
         # Copy everything inside the 'test-reports' folder
         run: |
           mkdir -p tests-report
           rsync -amL --include='*/' --include='test.xml' --include='test.log' --exclude='*' bazel-testlogs/ tests-report/
 
-      - name: Upload bundled consumer report
+      - name: Upload bundled test report
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: tests-report
           path: tests-report

docs.bzl

@@ -50,8 +50,8 @@ def _rewrite_needs_json_to_docs_sources(labels):
     out = []
     for x in labels:
         s = str(x)
-        if s.endswith("//:needs_json"):
-            out.append(s.replace("//:needs_json", "//:docs_sources"))
+        if s.endswith(":needs_json"):
+            out.append(s.replace(":needs_json", ":docs_sources"))
         else:
             out.append(s)
     return out
@@ -61,9 +61,9 @@ def _rewrite_needs_json_to_sourcelinks(labels):
     out = []
     for x in labels:
         s = str(x)
-        if s.endswith("//:needs_json"):
-            out.append(s.replace("//:needs_json", "//:sourcelinks_json"))
-        #Items which do not end up with '//:needs_json' shall not be appended to 'out'.
+        if s.endswith(":needs_json"):
+            out.append(s.replace(":needs_json", ":sourcelinks_json"))
+        #Items which do not end up with ':needs_json' shall not be appended to 'out'.
         #They are treated separately and are not related to source code linking.
     return out
 

pyproject.toml

@@ -109,5 +109,6 @@ filterwarnings = [
 ]
 
 pythonpath = [
+    ".",
     "src/extensions/",
 ]

src/tests/BUILD

@@ -0,0 +1,14 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+
+# Empty BUILD file ensures bazel will not see this directory when globbing

src/tests/README.md

@@ -13,105 +13,24 @@
 
 # Docs-As-Code Consumer Tests
 
-This test validates that changes to the docs-as-code system don't break downstream consumers.
-It tests both local changes and git-based overrides against real consumer repositories.
+# Testing
 
-## Use in CI
+![test levels in S-CORE docs-as-code](test-levels.drawio.svg)
 
-If you want to start the consumer tests on a PR inside `docs-as-code`, then all you have to do is comment
-`/consumer-test` on the PR and this should trigger them.
+## end-to-end tests
 
-## Quick Start
+*Also known as system, product, or black-box tests.*
 
-```bash
-# Create the virtual environment
-bazel run //:ide_support
-
-# Run with std. configuration
-.venv_docs/bin/python -m pytest -s src/tests
-
-# Run with more verbose output (up to -vvv)
-.venv_docs/bin/python -m pytest -s -v src/tests
-
-# Run specific repositories only
-.venv_docs/bin/python -m pytest -s src/tests --repo=score
-
-# Disable the persistent cache
-.venv_docs/bin/python -m pytest -s src/tests --disable-cache
-
-# Or combine both options
-.venv_docs/bin/python -m pytest -s src/tests --disable-cache --repo=score
-```
+This directory contains end-to-end tests, namely:
 
-## Verbosity Levels
+- `docs_bzl`: tests based on the public `docs()` macro. Must be executed via plain pytest (run via `.venv_docs`) as they execute real `bazel run`/`bazel build` internally. See [docs_bzl/README.md](docs_bzl/README.md) for details.
 
-The test suite supports different levels of output detail:
+- `downstream_compatibility`: *(formerly consumer tests)* Tests local changes and Git-based overrides against real consumer repositories. This provides broad, intentionally less-controlled coverage and helps detect breaking changes in the docs-as-code system before they affect downstream consumers.
 
-- **No flags**: Basic test results and summary table
-- **`-v`**: Shows detailed warnings breakdown by logger type
-- **`-vv`**: Adds full stdout/stderr output from build commands
-- **`-vvv`**: Streams build output in real-time (useful for debugging hanging builds)
+### Targets
 
-## Command Line Options
+You can query targets in this directory with:
 
-### `--disable-cache`
-Disabled persistent caching for clean testing cycle.
-
-**What the test normaly do:**
-- Uses `~/.cache/docs_as_code_consumer_tests` instead of temporary directories
-- Reuses cloned repositories between runs (with git updates)
-- Significantly speeds up subsequent test runs
-
-**This option disables the above mentioned behaviour and clones the repositories fresh**
-
-**When to use:** During development when you need to ensure testing is done on a fresh env.
-
-### `--repo`
-Filters which repositories to test.
-
-**Usage:**
 ```bash
-# Test only the 'score' repository
-.venv_docs/bin/python -m pytest -s src/tests --repo=score
-
-# Test multiple repositories
-.venv_docs/bin/python -m pytest -s src/tests --repo=score,module_template
-
-# Invalid repo names fall back to testing all repositories
-.venv_docs/bin/python -m pytest -s src/tests --repo=nonexistent
-```
-
-**Available repositories:** Check `REPOS_TO_TEST` in the test file for current list.
-
-## Currently tested repositories
-
-- [score](https://github.com/eclipse-score/score)
-- [process_description](https://github.com/eclipse-score/process_description)
-- [module_template](https://github.com/eclipse-score/module_template)
-
-## What Gets Tested
-
-For each repository, the test:
-1. Clones the consumer repository
-2. Tests with **local override** (your current changes)
-3. Tests with **git override** (current commit from remote)
-4. Runs build commands and test commands
-5. Analyzes warnings and build success
-
-## Example Development Workflow
-
-```bash
-# Create the virtual environment
-bazel run //:ide_support
-
-# First run - clones everything fresh
-.venv_docs/bin/python -m pytest -s -v src/tests --repo=score
-
-# Make changes to docs-as-code...
-
-# Subsequent runs - much faster due to caching
-.venv_docs/bin/python -m pytest -s -v src/tests --repo=score
-
-# Final validation - test all repos without cache
-.venv_docs/bin/python -m pytest -s -v src/tests --disable-cache
+bazel query 'kind(".*_test", //src/tests/...)'
 ```

src/tests/docs_bzl/README.md

@@ -0,0 +1,22 @@
+<!--
+  *******************************************************************************
+  Copyright (c) 2026 Contributors to the Eclipse Foundation
+
+  SPDX-License-Identifier: Apache-2.0
+  *******************************************************************************
+-->
+
+# `docs()` end-to-end tests
+
+These tests run **outside** Bazel with pytest and drive the public `docs()`
+macro through real `bazel run` / `bazel build` commands against small fixture
+packages in this directory. They test exactly what a user runs.
+
+Note that these tests run `bazel` commands, so they are slow. They need to be executed
+sequentially. Use sparingly.
+
+Run via:
+
+    pytest src/tests/docs_bzl -vv
+
+*(no venv is required)*

src/tests/docs_bzl/fixtures/basic/BUILD

@@ -0,0 +1,19 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+
+load("//:docs.bzl", "docs")
+
+docs(
+    source_dir = "docs",
+    test_sources = ["src/tests/docs_bzl/fixtures/basic"],
+)

src/tests/docs_bzl/fixtures/basic/docs/conf.py

@@ -0,0 +1,16 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+
+project = "Basic Test"
+project_url = "https://github.com/eclipse-score/docs-as-code"
+extensions = ["score_sphinx_bundle"]

src/tests/docs_bzl/fixtures/basic/docs/index.rst

@@ -0,0 +1,16 @@
+..
+   # *******************************************************************************
+   # Copyright (c) 2026 Contributors to the Eclipse Foundation
+   #
+   # See the NOTICE file(s) distributed with this work for additional
+   # information regarding copyright ownership.
+   #
+   # This program and the accompanying materials are made available under the
+   # terms of the Apache License Version 2.0 which is available at
+   # https://www.apache.org/licenses/LICENSE-2.0
+   #
+   # SPDX-License-Identifier: Apache-2.0
+   # *******************************************************************************
+
+Basic Test
+==========

src/tests/docs_bzl/fixtures/external_needs/consumer/BUILD

@@ -0,0 +1,23 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+
+load("//:docs.bzl", "docs")
+
+# Mounts the same-repo producer's needs.json so the :need: link in index.rst can
+# resolve. test_sources scopes the test-code-linker to this (test-less) package,
+# avoiding the workspace-wide bazel-testlogs scan during the html build.
+docs(
+    source_dir = "docs",
+    data = ["//src/tests/docs_bzl/fixtures/external_needs/producer:needs_json"],
+    test_sources = ["src/tests/docs_bzl/fixtures/external_needs/consumer"],
+)

src/tests/docs_bzl/fixtures/external_needs/consumer/docs/conf.py

@@ -0,0 +1,28 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+
+import os
+
+project = "External Needs Consumer"
+project_url = "https://example.invalid/external-needs-consumer"
+extensions = ["score_sphinx_bundle"]
+
+# FIXME:
+# sphinx-needs refuses to add an external need whose type is not registered in
+# this project's needs_types, so the consumer must declare the producer's
+# `test_req` type. score_metamodel's own checks skip external needs, so the
+# id/parts rules never run here. Loaded via the score_metamodel_yaml config
+# value because docs(metamodel=...) does not reach the build target.
+score_metamodel_yaml = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "metamodel.yaml"
+)

src/tests/docs_bzl/fixtures/external_needs/consumer/docs/index.rst

@@ -0,0 +1,19 @@
+..
+   # *******************************************************************************
+   # Copyright (c) 2026 Contributors to the Eclipse Foundation
+   #
+   # See the NOTICE file(s) distributed with this work for additional
+   # information regarding copyright ownership.
+   #
+   # This program and the accompanying materials are made available under the
+   # terms of the Apache License Version 2.0 which is available at
+   # https://www.apache.org/licenses/LICENSE-2.0
+   #
+   # SPDX-License-Identifier: Apache-2.0
+   # *******************************************************************************
+
+External Needs Consumer
+=======================
+
+This document links to a need defined in the producer package:
+:need:`test_req__producer__demo`.