diff --git a/.github/workflows/consumer_test.yml b/.github/workflows/downstream_compatibility.yml similarity index 92% rename from .github/workflows/consumer_test.yml rename to .github/workflows/downstream_compatibility.yml index 680fda38b..27cb5ac91 100644 --- a/.github/workflows/consumer_test.yml +++ b/.github/workflows/downstream_compatibility.yml @@ -11,7 +11,7 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -name: Consumer Tests +name: Downstream Compatibility on: workflow_call: pull_request: @@ -48,7 +48,7 @@ jobs: run: bazel run //:ide_support - name: Run Consumer tests - run: .venv_docs/bin/python -m pytest -vv -s src/tests/test_consumer.py -k "$CONSUMER" + run: .venv_docs/bin/python -m pytest -vv -s src/tests/downstream_compatibility -k "$CONSUMER" env: FORCE_COLOR: "1" TERM: xterm-256color diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d7fab5c3d..8773ef2dc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,6 +25,7 @@ jobs: uses: step-security/harden-runner@v2.18.0 with: egress-policy: audit + - name: Checkout repository (Handle all events) uses: actions/checkout@v4.2.2 with: @@ -38,13 +39,22 @@ jobs: repository-cache: true bazelisk-cache: true cache-save: ${{ github.event_name == 'push' }} - - name: Run test targets - run: | - bazel run --lockfile_mode=error //:ide_support - bazel test --lockfile_mode=error //... - - name: Prepare bundled consumer report - if: always() + - name: Setup venv + run: bazel run --lockfile_mode=error //:ide_support + + - name: Run docs_bzl tests + run: .venv_docs/bin/python -m pytest -vv src/tests/docs_bzl + + - name: Run bazel test targets + run: bazel test --lockfile_mode=error //... --build_tests_only + + - name: Build all targets + # Skip tests, since some are non buildable negative tests + run: bazel build --lockfile_mode=error //... -- -//src/tests/... + + - name: Prepare bundled test report + if: ! cancelled() # Creating tests-report directory # Follow Symlinks via '-L' to copy correctly # Copy everything inside the 'test-reports' folder @@ -52,9 +62,9 @@ jobs: mkdir -p tests-report rsync -amL --include='*/' --include='test.xml' --include='test.log' --exclude='*' bazel-testlogs/ tests-report/ - - name: Upload bundled consumer report + - name: Upload bundled test report if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: tests-report path: tests-report diff --git a/docs.bzl b/docs.bzl index a03d54e67..30c0cdaa3 100644 --- a/docs.bzl +++ b/docs.bzl @@ -50,8 +50,8 @@ def _rewrite_needs_json_to_docs_sources(labels): out = [] for x in labels: s = str(x) - if s.endswith("//:needs_json"): - out.append(s.replace("//:needs_json", "//:docs_sources")) + if s.endswith(":needs_json"): + out.append(s.replace(":needs_json", ":docs_sources")) else: out.append(s) return out @@ -61,9 +61,9 @@ def _rewrite_needs_json_to_sourcelinks(labels): out = [] for x in labels: s = str(x) - if s.endswith("//:needs_json"): - out.append(s.replace("//:needs_json", "//:sourcelinks_json")) - #Items which do not end up with '//:needs_json' shall not be appended to 'out'. + if s.endswith(":needs_json"): + out.append(s.replace(":needs_json", ":sourcelinks_json")) + #Items which do not end up with ':needs_json' shall not be appended to 'out'. #They are treated separately and are not related to source code linking. return out diff --git a/pyproject.toml b/pyproject.toml index 277775e0d..cb555641f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,5 +109,6 @@ filterwarnings = [ ] pythonpath = [ + ".", "src/extensions/", ] diff --git a/src/tests/BUILD b/src/tests/BUILD new file mode 100644 index 000000000..a9f47d3de --- /dev/null +++ b/src/tests/BUILD @@ -0,0 +1,14 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# Empty BUILD file ensures bazel will not see this directory when globbing diff --git a/src/tests/README.md b/src/tests/README.md index 05e0d85c1..060487730 100644 --- a/src/tests/README.md +++ b/src/tests/README.md @@ -13,105 +13,24 @@ # Docs-As-Code Consumer Tests -This test validates that changes to the docs-as-code system don't break downstream consumers. -It tests both local changes and git-based overrides against real consumer repositories. +# Testing -## Use in CI +![test levels in S-CORE docs-as-code](test-levels.drawio.svg) -If you want to start the consumer tests on a PR inside `docs-as-code`, then all you have to do is comment -`/consumer-test` on the PR and this should trigger them. +## end-to-end tests -## Quick Start +*Also known as system, product, or black-box tests.* -```bash -# Create the virtual environment -bazel run //:ide_support - -# Run with std. configuration -.venv_docs/bin/python -m pytest -s src/tests - -# Run with more verbose output (up to -vvv) -.venv_docs/bin/python -m pytest -s -v src/tests - -# Run specific repositories only -.venv_docs/bin/python -m pytest -s src/tests --repo=score - -# Disable the persistent cache -.venv_docs/bin/python -m pytest -s src/tests --disable-cache - -# Or combine both options -.venv_docs/bin/python -m pytest -s src/tests --disable-cache --repo=score -``` +This directory contains end-to-end tests, namely: -## Verbosity Levels +- `docs_bzl`: tests based on the public `docs()` macro. Must be executed via plain pytest (run via `.venv_docs`) as they execute real `bazel run`/`bazel build` internally. See [docs_bzl/README.md](docs_bzl/README.md) for details. -The test suite supports different levels of output detail: +- `downstream_compatibility`: *(formerly consumer tests)* Tests local changes and Git-based overrides against real consumer repositories. This provides broad, intentionally less-controlled coverage and helps detect breaking changes in the docs-as-code system before they affect downstream consumers. -- **No flags**: Basic test results and summary table -- **`-v`**: Shows detailed warnings breakdown by logger type -- **`-vv`**: Adds full stdout/stderr output from build commands -- **`-vvv`**: Streams build output in real-time (useful for debugging hanging builds) +### Targets -## Command Line Options +You can query targets in this directory with: -### `--disable-cache` -Disabled persistent caching for clean testing cycle. - -**What the test normaly do:** -- Uses `~/.cache/docs_as_code_consumer_tests` instead of temporary directories -- Reuses cloned repositories between runs (with git updates) -- Significantly speeds up subsequent test runs - -**This option disables the above mentioned behaviour and clones the repositories fresh** - -**When to use:** During development when you need to ensure testing is done on a fresh env. - -### `--repo` -Filters which repositories to test. - -**Usage:** ```bash -# Test only the 'score' repository -.venv_docs/bin/python -m pytest -s src/tests --repo=score - -# Test multiple repositories -.venv_docs/bin/python -m pytest -s src/tests --repo=score,module_template - -# Invalid repo names fall back to testing all repositories -.venv_docs/bin/python -m pytest -s src/tests --repo=nonexistent -``` - -**Available repositories:** Check `REPOS_TO_TEST` in the test file for current list. - -## Currently tested repositories - -- [score](https://github.com/eclipse-score/score) -- [process_description](https://github.com/eclipse-score/process_description) -- [module_template](https://github.com/eclipse-score/module_template) - -## What Gets Tested - -For each repository, the test: -1. Clones the consumer repository -2. Tests with **local override** (your current changes) -3. Tests with **git override** (current commit from remote) -4. Runs build commands and test commands -5. Analyzes warnings and build success - -## Example Development Workflow - -```bash -# Create the virtual environment -bazel run //:ide_support - -# First run - clones everything fresh -.venv_docs/bin/python -m pytest -s -v src/tests --repo=score - -# Make changes to docs-as-code... - -# Subsequent runs - much faster due to caching -.venv_docs/bin/python -m pytest -s -v src/tests --repo=score - -# Final validation - test all repos without cache -.venv_docs/bin/python -m pytest -s -v src/tests --disable-cache +bazel query 'kind(".*_test", //src/tests/...)' ``` diff --git a/src/tests/docs_bzl/README.md b/src/tests/docs_bzl/README.md new file mode 100644 index 000000000..b483b4764 --- /dev/null +++ b/src/tests/docs_bzl/README.md @@ -0,0 +1,22 @@ + + +# `docs()` end-to-end tests + +These tests run **outside** Bazel with pytest and drive the public `docs()` +macro through real `bazel run` / `bazel build` commands against small fixture +packages in this directory. They test exactly what a user runs. + +Note that these tests run `bazel` commands, so they are slow. They need to be executed +sequentially. Use sparingly. + +Run via: + + pytest src/tests/docs_bzl -vv + +*(no venv is required)* diff --git a/src/tests/docs_bzl/fixtures/basic/BUILD b/src/tests/docs_bzl/fixtures/basic/BUILD new file mode 100644 index 000000000..a67698dba --- /dev/null +++ b/src/tests/docs_bzl/fixtures/basic/BUILD @@ -0,0 +1,19 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("//:docs.bzl", "docs") + +docs( + source_dir = "docs", + test_sources = ["src/tests/docs_bzl/fixtures/basic"], +) diff --git a/src/tests/docs_bzl/fixtures/basic/docs/conf.py b/src/tests/docs_bzl/fixtures/basic/docs/conf.py new file mode 100644 index 000000000..9b0fd6e88 --- /dev/null +++ b/src/tests/docs_bzl/fixtures/basic/docs/conf.py @@ -0,0 +1,16 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +project = "Basic Test" +project_url = "https://github.com/eclipse-score/docs-as-code" +extensions = ["score_sphinx_bundle"] diff --git a/src/tests/docs_bzl/fixtures/basic/docs/index.rst b/src/tests/docs_bzl/fixtures/basic/docs/index.rst new file mode 100644 index 000000000..cc8afd54b --- /dev/null +++ b/src/tests/docs_bzl/fixtures/basic/docs/index.rst @@ -0,0 +1,16 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +Basic Test +========== diff --git a/src/tests/docs_bzl/fixtures/external_needs/consumer/BUILD b/src/tests/docs_bzl/fixtures/external_needs/consumer/BUILD new file mode 100644 index 000000000..977e96715 --- /dev/null +++ b/src/tests/docs_bzl/fixtures/external_needs/consumer/BUILD @@ -0,0 +1,23 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("//:docs.bzl", "docs") + +# Mounts the same-repo producer's needs.json so the :need: link in index.rst can +# resolve. test_sources scopes the test-code-linker to this (test-less) package, +# avoiding the workspace-wide bazel-testlogs scan during the html build. +docs( + source_dir = "docs", + data = ["//src/tests/docs_bzl/fixtures/external_needs/producer:needs_json"], + test_sources = ["src/tests/docs_bzl/fixtures/external_needs/consumer"], +) diff --git a/src/tests/docs_bzl/fixtures/external_needs/consumer/docs/conf.py b/src/tests/docs_bzl/fixtures/external_needs/consumer/docs/conf.py new file mode 100644 index 000000000..9e5c5d250 --- /dev/null +++ b/src/tests/docs_bzl/fixtures/external_needs/consumer/docs/conf.py @@ -0,0 +1,28 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +import os + +project = "External Needs Consumer" +project_url = "https://example.invalid/external-needs-consumer" +extensions = ["score_sphinx_bundle"] + +# FIXME: +# sphinx-needs refuses to add an external need whose type is not registered in +# this project's needs_types, so the consumer must declare the producer's +# `test_req` type. score_metamodel's own checks skip external needs, so the +# id/parts rules never run here. Loaded via the score_metamodel_yaml config +# value because docs(metamodel=...) does not reach the build target. +score_metamodel_yaml = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "metamodel.yaml" +) diff --git a/src/tests/docs_bzl/fixtures/external_needs/consumer/docs/index.rst b/src/tests/docs_bzl/fixtures/external_needs/consumer/docs/index.rst new file mode 100644 index 000000000..17f7631a6 --- /dev/null +++ b/src/tests/docs_bzl/fixtures/external_needs/consumer/docs/index.rst @@ -0,0 +1,19 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +External Needs Consumer +======================= + +This document links to a need defined in the producer package: +:need:`test_req__producer__demo`. diff --git a/src/tests/docs_bzl/fixtures/external_needs/consumer/docs/metamodel.yaml b/src/tests/docs_bzl/fixtures/external_needs/consumer/docs/metamodel.yaml new file mode 100644 index 000000000..dc4e1651a --- /dev/null +++ b/src/tests/docs_bzl/fixtures/external_needs/consumer/docs/metamodel.yaml @@ -0,0 +1,29 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +needs_types: + test_req: + title: Test Requirement + prefix: test_req__ + parts: 3 + mandatory_options: + id: ^test_req__[0-9a-zA-Z_]*$ + status: ^(draft|valid)$ + optional_options: + # These are all sphinx-needs default fields. Listing them keeps the + # metamodel/default option overlap exactly at the set score_metamodel + # treats as intentional ({id, tags, status, content, template}), which + # avoids a setup-time warning that -W would turn into a build error. + tags: .* + content: .* + template: .* +links: {} diff --git a/src/tests/docs_bzl/fixtures/external_needs/producer/BUILD b/src/tests/docs_bzl/fixtures/external_needs/producer/BUILD new file mode 100644 index 000000000..352903a86 --- /dev/null +++ b/src/tests/docs_bzl/fixtures/external_needs/producer/BUILD @@ -0,0 +1,22 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("//:docs.bzl", "docs") + +# FIXME: +# The custom metamodel is supplied via score_metamodel_yaml in docs/conf.py +# rather than docs(metamodel=...), because the metamodel= wiring does not reach +# the :needs_json target in the current docs.bzl (regression from #484). +docs( + source_dir = "docs", +) diff --git a/src/tests/docs_bzl/fixtures/external_needs/producer/docs/conf.py b/src/tests/docs_bzl/fixtures/external_needs/producer/docs/conf.py new file mode 100644 index 000000000..5bdbc3015 --- /dev/null +++ b/src/tests/docs_bzl/fixtures/external_needs/producer/docs/conf.py @@ -0,0 +1,38 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +import os + +project = "External Needs Producer" +project_url = "https://example.invalid/external-needs-producer" +extensions = ["score_sphinx_bundle"] + +# sphinx-needs writes this as the needs.json `current_version`. A consumer that +# mounts this needs.json resolves needs under this version key, so it must be +# non-empty (an unset version yields an empty current_version that sphinx-needs +# rejects with "No version defined"). +version = "main" + +# docs(metamodel=...) does not reach the :needs_json (sphinx_docs) target in the +# current docs.bzl, so we load the custom metamodel via the officially-supported +# score_metamodel_yaml config value. The YAML lives next to this conf.py so it is +# globbed into :docs_sources and materialized in the sandbox. +score_metamodel_yaml = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "metamodel.yaml" +) + +# The producer need id `test_req__producer__demo` has three "__"-parts. The +# id_contains_feature check requires the feature part ("producer") to appear in +# the docname or in required_in_id; the fixture lives in index.rst, so we declare +# it here. +required_in_id = ["producer"] diff --git a/src/tests/docs_bzl/fixtures/external_needs/producer/docs/index.rst b/src/tests/docs_bzl/fixtures/external_needs/producer/docs/index.rst new file mode 100644 index 000000000..8e5a06b4f --- /dev/null +++ b/src/tests/docs_bzl/fixtures/external_needs/producer/docs/index.rst @@ -0,0 +1,20 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +External Needs Producer +======================= + +.. test_req:: Producer demo requirement + :id: test_req__producer__demo + :status: valid diff --git a/src/tests/docs_bzl/fixtures/external_needs/producer/docs/metamodel.yaml b/src/tests/docs_bzl/fixtures/external_needs/producer/docs/metamodel.yaml new file mode 100644 index 000000000..dc4e1651a --- /dev/null +++ b/src/tests/docs_bzl/fixtures/external_needs/producer/docs/metamodel.yaml @@ -0,0 +1,29 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +needs_types: + test_req: + title: Test Requirement + prefix: test_req__ + parts: 3 + mandatory_options: + id: ^test_req__[0-9a-zA-Z_]*$ + status: ^(draft|valid)$ + optional_options: + # These are all sphinx-needs default fields. Listing them keeps the + # metamodel/default option overlap exactly at the set score_metamodel + # treats as intentional ({id, tags, status, content, template}), which + # avoids a setup-time warning that -W would turn into a build error. + tags: .* + content: .* + template: .* +links: {} diff --git a/src/tests/docs_bzl/fixtures/metamodel_violation/BUILD b/src/tests/docs_bzl/fixtures/metamodel_violation/BUILD new file mode 100644 index 000000000..352903a86 --- /dev/null +++ b/src/tests/docs_bzl/fixtures/metamodel_violation/BUILD @@ -0,0 +1,22 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("//:docs.bzl", "docs") + +# FIXME: +# The custom metamodel is supplied via score_metamodel_yaml in docs/conf.py +# rather than docs(metamodel=...), because the metamodel= wiring does not reach +# the :needs_json target in the current docs.bzl (regression from #484). +docs( + source_dir = "docs", +) diff --git a/src/tests/docs_bzl/fixtures/metamodel_violation/docs/conf.py b/src/tests/docs_bzl/fixtures/metamodel_violation/docs/conf.py new file mode 100644 index 000000000..557f0d7fe --- /dev/null +++ b/src/tests/docs_bzl/fixtures/metamodel_violation/docs/conf.py @@ -0,0 +1,27 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +import os + +project = "Metamodel Check" +project_url = "https://example.invalid/metamodel-check" +extensions = ["score_sphinx_bundle"] + +# NOTE: docs(metamodel=...) does not reach the :needs_json (sphinx_docs) target +# in the current docs.bzl (the metamodel_opts/metamodel_data wiring was dropped +# in commit 970f2604 / #484). We therefore load the custom metamodel via the +# officially-supported score_metamodel_yaml config value. The YAML lives next to +# this conf.py so it is globbed into :docs_sources and materialized in the sandbox. +score_metamodel_yaml = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "metamodel.yaml" +) diff --git a/src/tests/docs_bzl/fixtures/metamodel_violation/docs/index.rst b/src/tests/docs_bzl/fixtures/metamodel_violation/docs/index.rst new file mode 100644 index 000000000..fa0a6d725 --- /dev/null +++ b/src/tests/docs_bzl/fixtures/metamodel_violation/docs/index.rst @@ -0,0 +1,19 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +Metamodel Check +=============== + +.. test_req:: A requirement missing its status + :id: test_req__demo_missing diff --git a/src/tests/docs_bzl/fixtures/metamodel_violation/docs/metamodel.yaml b/src/tests/docs_bzl/fixtures/metamodel_violation/docs/metamodel.yaml new file mode 100644 index 000000000..386f815e4 --- /dev/null +++ b/src/tests/docs_bzl/fixtures/metamodel_violation/docs/metamodel.yaml @@ -0,0 +1,30 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +needs_types: + test_req: + title: Test Requirement + prefix: test_req__ + parts: 2 + mandatory_options: + id: ^test_req__[0-9a-zA-Z_]*$ + status: ^(draft|valid)$ + optional_options: + # FIXME: + # These are all sphinx-needs default fields. Listing them keeps the + # metamodel/default option overlap exactly at the set score_metamodel + # treats as intentional ({id, tags, status, content, template}), which + # avoids a setup-time warning that -W would turn into a build error. + tags: .* + content: .* + template: .* +links: {} diff --git a/src/tests/docs_bzl/helpers.py b/src/tests/docs_bzl/helpers.py new file mode 100644 index 000000000..13611ed6f --- /dev/null +++ b/src/tests/docs_bzl/helpers.py @@ -0,0 +1,142 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Helpers for Category-2 e2e tests: run real bazel against in-repo fixtures.""" + +import json +import shutil +import subprocess +import time +from dataclasses import dataclass +from pathlib import Path + +from src.helper_lib import find_git_root + +FIXTURES = Path(__file__).parent / "fixtures" +GIT_ROOT = find_git_root() +assert GIT_ROOT +FIXTURES_BAZEL = "//" + str(FIXTURES.relative_to(GIT_ROOT)) + + +def repo_root() -> Path: + root = find_git_root() + assert root is not None, "git root not found" + return root + + +def run_bazel(args: list[str], expect_error: bool = False): + start_time = time.time() + cmd = ["bazel", *args] + cmd_str = " ".join(cmd) + + p = subprocess.run( + cmd, + cwd=repo_root(), + capture_output=True, + text=True, + ) + end_time = time.time() + print(f"Running 'bazel {' '.join(args)}' took {end_time - start_time:.4f} seconds") + + if expect_error and p.returncode == 0: + raise RuntimeError( + f"{cmd_str} was expected to fail but succeeded\n" + f"stdout:\n{p.stdout}\n" + f"stderr:\n{p.stderr}" + ) + + if not expect_error and p.returncode != 0: + raise RuntimeError( + f"bazel {args} failed with exit code {p.returncode}\n" + f"stdout:\n{p.stdout}\n" + f"stderr:\n{p.stderr}" + ) + + return p + + +@dataclass +class RunResult: + stdout: str + stderr: str + build_dir: Path + artifacts: dict[str, Path] | None + + +def run_fixture( + bazel_cmd: str, + fixture: str, + target: str, + expect_error: bool = False, +) -> RunResult: + """Run bazel in a fixture subdirectory.""" + assert bazel_cmd in ("build", "run"), "only build and run are supported" + assert target.startswith(":"), "target must be relative to fixture" + + if not (FIXTURES / fixture).is_dir(): + raise ValueError(f"fixture {fixture} does not exist in {FIXTURES}") + + # Split logs by bazel commands visually without adding any fancy formatting dependency + print("") + + full_target = FIXTURES_BAZEL + "/" + fixture + target + + clean_build(FIXTURES / fixture) + + p1 = run_bazel([bazel_cmd, full_target], expect_error=expect_error) + + if bazel_cmd == "build": + # Generic solution needs to use cquery. + # That extra cquery is about 0,5 seconds, which sounds horrible for testing, + # but considering how slow bazel is anyway... + RUN_CQUERY = False + if RUN_CQUERY: + p2 = run_bazel(["cquery", "--output=files", full_target]) + artifacts_dir = Path(p2.stdout.strip().splitlines()[0]) + else: + assert GIT_ROOT + # e.g. bazel-out/k8-fastbuild/bin/src/tests/docs_bzl/fixtures/external_needs/producer/needs_json/_build/needs + artifacts_dir = ( + GIT_ROOT + / "bazel-out/k8-fastbuild/bin/" + / FIXTURES.relative_to(GIT_ROOT) + / fixture + / target[1:] + / "_build/needs" + ) + + assert artifacts_dir.is_dir(), ( + f"expected output artifacts to be a directory, got {artifacts_dir}" + ) + artifacts = {f.name: f for f in artifacts_dir.iterdir()} + else: + artifacts = None + + return RunResult( + stdout=p1.stdout, + stderr=p1.stderr, + build_dir=FIXTURES / fixture / "_build", + artifacts=artifacts, + ) + + +def clean_build(path: Path) -> None: + """Delete the _build directory in a path, if it exists.""" + shutil.rmtree(path / "_build", ignore_errors=True) + + +def load_needs(needs_json: Path): + data = json.loads(needs_json.read_text(encoding="utf-8")) + needs: dict[str, object] = {} + for version in data.get("versions", {}).values(): + needs.update(version.get("needs", {})) + return needs diff --git a/src/tests/docs_bzl/test_docs_bzl.py b/src/tests/docs_bzl/test_docs_bzl.py new file mode 100644 index 000000000..701536777 --- /dev/null +++ b/src/tests/docs_bzl/test_docs_bzl.py @@ -0,0 +1,74 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +""" +end-to-end tests for the public docs() macro. +See README.md in this directory for usage instructions. +""" + +from pathlib import Path + +from src.tests.docs_bzl.helpers import ( + load_needs, + run_fixture, +) + +FIXTURES = Path(__file__).parent / "fixtures" + + +def test_basic_docs_builds_html(): + result = run_fixture("run", "basic", ":docs") + + index_html = result.build_dir / "index.html" + + assert "Basic Test" in index_html.read_text(encoding="utf-8") + + +def test_metamodel_violation_fails_build(): + result = run_fixture( + "build", "metamodel_violation", ":needs_json", expect_error=True + ) + assert "is missing required attribute" in result.stderr + + +def test_producer_needs_json_contains_local_need(): + """ + The producer must export its own need + """ + result = run_fixture("build", "external_needs/producer", ":needs_json") + assert result.artifacts + + needs_json = result.artifacts["needs.json"] + needs = load_needs(needs_json) + + assert "test_req__producer__demo" in needs, sorted(needs) + + +def test_consumer_link_resolves(): + """ + The consumer must resolve the producer's need as an external link in its HTML output. + + Exit 0 under -W already proves no "not found" warning fired; + the html-content assert is the stronger guard: the producer id must + appear as a resolved external link (href into the producer's base_url), not + just as literal text. + """ + result = run_fixture("run", "external_needs/consumer", ":docs") + + index = result.build_dir / "index.html" + html = index.read_text(encoding="utf-8") + + # A resolved external link points at the producer's base_url + fragment. + assert "external-needs-producer/main/index.html#test_req__producer__demo" in html, ( + html + ) diff --git a/src/tests/downstream_compatibility/README.md b/src/tests/downstream_compatibility/README.md new file mode 100644 index 000000000..05e0d85c1 --- /dev/null +++ b/src/tests/downstream_compatibility/README.md @@ -0,0 +1,117 @@ + + +# Docs-As-Code Consumer Tests + +This test validates that changes to the docs-as-code system don't break downstream consumers. +It tests both local changes and git-based overrides against real consumer repositories. + +## Use in CI + +If you want to start the consumer tests on a PR inside `docs-as-code`, then all you have to do is comment +`/consumer-test` on the PR and this should trigger them. + +## Quick Start + +```bash +# Create the virtual environment +bazel run //:ide_support + +# Run with std. configuration +.venv_docs/bin/python -m pytest -s src/tests + +# Run with more verbose output (up to -vvv) +.venv_docs/bin/python -m pytest -s -v src/tests + +# Run specific repositories only +.venv_docs/bin/python -m pytest -s src/tests --repo=score + +# Disable the persistent cache +.venv_docs/bin/python -m pytest -s src/tests --disable-cache + +# Or combine both options +.venv_docs/bin/python -m pytest -s src/tests --disable-cache --repo=score +``` + +## Verbosity Levels + +The test suite supports different levels of output detail: + +- **No flags**: Basic test results and summary table +- **`-v`**: Shows detailed warnings breakdown by logger type +- **`-vv`**: Adds full stdout/stderr output from build commands +- **`-vvv`**: Streams build output in real-time (useful for debugging hanging builds) + +## Command Line Options + +### `--disable-cache` +Disabled persistent caching for clean testing cycle. + +**What the test normaly do:** +- Uses `~/.cache/docs_as_code_consumer_tests` instead of temporary directories +- Reuses cloned repositories between runs (with git updates) +- Significantly speeds up subsequent test runs + +**This option disables the above mentioned behaviour and clones the repositories fresh** + +**When to use:** During development when you need to ensure testing is done on a fresh env. + +### `--repo` +Filters which repositories to test. + +**Usage:** +```bash +# Test only the 'score' repository +.venv_docs/bin/python -m pytest -s src/tests --repo=score + +# Test multiple repositories +.venv_docs/bin/python -m pytest -s src/tests --repo=score,module_template + +# Invalid repo names fall back to testing all repositories +.venv_docs/bin/python -m pytest -s src/tests --repo=nonexistent +``` + +**Available repositories:** Check `REPOS_TO_TEST` in the test file for current list. + +## Currently tested repositories + +- [score](https://github.com/eclipse-score/score) +- [process_description](https://github.com/eclipse-score/process_description) +- [module_template](https://github.com/eclipse-score/module_template) + +## What Gets Tested + +For each repository, the test: +1. Clones the consumer repository +2. Tests with **local override** (your current changes) +3. Tests with **git override** (current commit from remote) +4. Runs build commands and test commands +5. Analyzes warnings and build success + +## Example Development Workflow + +```bash +# Create the virtual environment +bazel run //:ide_support + +# First run - clones everything fresh +.venv_docs/bin/python -m pytest -s -v src/tests --repo=score + +# Make changes to docs-as-code... + +# Subsequent runs - much faster due to caching +.venv_docs/bin/python -m pytest -s -v src/tests --repo=score + +# Final validation - test all repos without cache +.venv_docs/bin/python -m pytest -s -v src/tests --disable-cache +``` diff --git a/src/tests/conftest.py b/src/tests/downstream_compatibility/conftest.py similarity index 100% rename from src/tests/conftest.py rename to src/tests/downstream_compatibility/conftest.py diff --git a/src/tests/test_consumer.py b/src/tests/downstream_compatibility/test_downstream_compatibility.py similarity index 95% rename from src/tests/test_consumer.py rename to src/tests/downstream_compatibility/test_downstream_compatibility.py index 046c35c67..d22cb77dd 100644 --- a/src/tests/test_consumer.py +++ b/src/tests/downstream_compatibility/test_downstream_compatibility.py @@ -11,14 +11,14 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* """ -Consumer tests: verify that downstream repos build successfully against this branch. +Downstream compatibility tests: verify that downstream repos build successfully against this branch. Via Python (requires ide_support to have been run first): bazel run //:ide_support # once, to set up .venv_docs - python -m pytest src/tests/test_consumer.py -s - python -m pytest src/tests/test_consumer.py -k "score and local" - python -m pytest src/tests/test_consumer.py -k "process_description" - python -m pytest src/tests/test_consumer.py --disable-cache + python -m pytest src/tests/downstream_compatibility -s + python -m pytest src/tests/downstream_compatibility -k "score and local" + python -m pytest src/tests/downstream_compatibility -k "process_description" + python -m pytest src/tests/downstream_compatibility --disable-cache Known non-passing tests are sometimes marked xfail and do not count as failures. """ @@ -66,7 +66,7 @@ def __init__( self.commands = commands else: # The minimum is to ensure that the repo can build against the current score_docs_as_code branch. - # If docs works, then usually docs_checl and needs_json will work as well. + # If docs works, then usually docs_check and needs_json will work as well. self.commands = ["bazel run //:docs"] @@ -222,6 +222,8 @@ def _ensure_repo(repo_path: Path, git_url: str, use_cache: bool) -> None: def _cleanup_before_cmd(cwd: Path, cmd: str) -> None: # ubproject.toml is created by :docs + # Delete in cwd (repo root) and any subdir to avoid stale config from previous runs + (cwd / "ubproject.toml").unlink(missing_ok=True) for p in cwd.glob("*/ubproject.toml"): p.unlink() diff --git a/src/tests/test-levels.drawio.svg b/src/tests/test-levels.drawio.svg new file mode 100644 index 000000000..e7c6d43dc --- /dev/null +++ b/src/tests/test-levels.drawio.svg @@ -0,0 +1,332 @@ + + + + + + + + + + +
+
+
+ Extension Test Scope +
+
+
+ + + Extension Test Scope + + + + + + + + + + + +
+
+
+ Application Test Scope +
+
+
+ + + Application Test Scope + + + + + + + + + + + +
+
+
+ Unit Test Scope +
+
+
+ + + Unit Test Scope + + + + + + + + + + + + + + + +
+
+
+ Downstream-Compatiblity Test Scope +
+
+
+ + + Downstream-Compatiblity Test Scope + + + + + + + + + + + +
+
+
+ Module Test Scope +
+
+
+ + + Module Test Scope + + + + + + + + + + + +
+
+
+ docs-bzl Scope +
+
+
+ + + docs-bzl Scope + + + + + + + + + + + +
+
+
+ docs-as-code Testing Scopes +
+
+
+ + + docs-as-code Testing Scopes + + + + + + + + + + + +
+
+
+ Each overlay shows which part of the architecture is exercised by a test level. +
+
+
+ + + Each overlay shows which part of the architecture is exercised by a test level. + + + + + + + + + + + +
+
+
+ MODULE.bazel +
+
+
+ + + MODULE.bazel + + + + + + + + + + + +
+
+
+ docs() macro +
+
+
+ + + docs() macro + + + + + + + + + + + + + + + +
+
+
+ Sphinx application +
+
+
+ + + Sphinx application + + + + + + + + + + + +
+
+
+ Python functions +
+
+
+ + + Python functions + + + + + + + + + + + +
+
+
+ HTML output +
+
+
+ + + HTML output + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ Downstream +
+ consumers +
+
+
+ + + Downstream... + + + + + + + + + + Text is not SVG - cannot display + + + +