Add test of documentation warnings. (#1857)

.circleci/config.yml

@@ -164,7 +164,7 @@ jobs:
           py_ver: <<parameters.py_ver>>
           tf_ver: <<parameters.tf_ver>>
           tfp_ver: <<parameters.tfp_ver>>
-          pytest_filter: not notebooks
+          pytest_filter: not notebooks and not docs
 
   notebook-test:
     parameters:
@@ -183,6 +183,17 @@ jobs:
           tfp_ver: <<parameters.tfp_ver>>
           pytest_filter: notebooks
 
+  docs-test:
+    docker:
+      # build-docs below use `max` version, so let's test with the same version we're going to use.
+      - image: cimg/python:<<pipeline.parameters.max_py_ver>>
+    steps:
+      - run_tests:
+          py_ver: <<pipeline.parameters.max_py_ver>>
+          tf_ver: <<pipeline.parameters.max_tf_ver>>
+          tfp_ver: <<pipeline.parameters.max_tfp_ver>>
+          pytest_filter: docs
+
   build-docs:
     docker:
       # At the time of writing we cannot generate documentation for type numpy with older versions,
@@ -329,6 +340,10 @@ workflows:
           filters:
             tags:
               only: /^v[0-9]+(\.[0-9]+)*(-rc[0-9]+)?/
+      - docs-test:
+          filters:
+            tags:
+              only: /^v[0-9]+(\.[0-9]+)*(-rc[0-9]+)?/
       - noop:
           name: fast-tests
           requires:
@@ -338,6 +353,7 @@ workflows:
             - type-check-max
             - format-check-min
             - format-check-max
+            - docs-test
       - unit-test:
           name: unit-test-min
           py_ver: <<pipeline.parameters.min_py_ver>>

doc/build_docs.py

@@ -38,6 +38,24 @@
 _DOCTREE_TMP = _TMP / "doctree"
 
 
+def _create_fake_notebook(destination_relative_path: Path, max_notebooks: int) -> None:
+    print(f"Generating fake, due to --max_notebooks={max_notebooks}")
+
+    destination = _NOTEBOOKS_TMP / destination_relative_path
+    title = f"Fake {destination.name}"
+    title_line = "#" * len(title)
+
+    destination.write_text(
+        f"""{title}
+{title_line}
+
+Fake {destination.name} due to::
+
+   --max_notebooks={max_notebooks}
+"""
+    )
+
+
 def _build_notebooks(max_notebooks: Optional[int]) -> None:
     # Building the notebooks is really slow. Let's time it so we know which notebooks we can /
     # should optimise.
@@ -50,27 +68,28 @@ def _build_notebooks(max_notebooks: Optional[int]) -> None:
         print("Building:", source_path)
         print("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
 
-        if max_notebooks is not None and i >= max_notebooks:
-            print(f"...skipping due to --max_notebooks={max_notebooks}")
-            continue
-
         source_relative_path = source_path.relative_to(_NOTEBOOKS_TMP)
         destination_relative_path = source_relative_path
         while destination_relative_path.suffix:  # .pct.py has several suffixes. Remove all of them.
             destination_relative_path = destination_relative_path.with_suffix("")
         destination_relative_path = destination_relative_path.with_suffix(".ipynb")
-        subprocess.run(
-            [
-                "jupytext",
-                "--execute",
-                "--to",
-                "notebook",
-                "-o",
-                str(destination_relative_path),
-                str(source_relative_path),
-            ],
-            cwd=_NOTEBOOKS_TMP,
-        ).check_returncode()
+
+        if max_notebooks is None or i < max_notebooks:
+            subprocess.run(
+                [
+                    "jupytext",
+                    "--execute",
+                    "--to",
+                    "notebook",
+                    "-o",
+                    str(destination_relative_path),
+                    str(source_relative_path),
+                ],
+                cwd=_NOTEBOOKS_TMP,
+            ).check_returncode()
+        else:
+            _create_fake_notebook(destination_relative_path, max_notebooks)
+
         after = perf_counter()
         timings.append((after - before, source_relative_path))
 
@@ -100,6 +119,13 @@ def main() -> None:
         type=int,
         help="Limit number of notebooks built to this number. Useful when debugging.",
     )
+    parser.add_argument(
+        "--fail_on_warning",
+        "--fail-on-warning",
+        default=False,
+        action="store_true",
+        help="If set, crash if there were any warnings while generating documentation.",
+    )
     args = parser.parse_args()
     branch = Branch(args.branch)
     dest = args.destination
@@ -113,17 +139,25 @@ def main() -> None:
     (_BUILD_TMP / "build_version.txt").write_text(branch.version)
     _build_notebooks(args.max_notebooks)
     generate_module_rst(gpflow, _BUILD_TMP / "api")
-    subprocess.run(
-        [
-            "sphinx-build",
-            "-b",
-            "html",
-            "-d",
-            str(_DOCTREE_TMP),
-            str(_BUILD_TMP),
-            str(version_dest),
-        ]
-    ).check_returncode()
+
+    sphinx_commands = [
+        "sphinx-build",
+        "-b",
+        "html",
+        "-d",
+        str(_DOCTREE_TMP),
+        str(_BUILD_TMP),
+        str(version_dest),
+    ]
+    if args.fail_on_warning:
+        sphinx_commands.extend(
+            [
+                "-W",
+                "--keep-going",
+            ]
+        )
+
+    subprocess.run(sphinx_commands).check_returncode()
 
 
 if __name__ == "__main__":

gpflow/experimental/check_shapes/argument_ref.py

@@ -96,7 +96,7 @@ def __repr__(self) -> str:
 
 @dataclass(frozen=True)  # type: ignore
 class DelegatingArgumentRef(ArgumentRef):
-    """ Abstract base class for `ArgumentRef`s the delegates to a source. """
+    """ Abstract base class for :class:`ArgumentRef`\ s that delegates to a source. """
 
     source: ArgumentRef
 

gpflow/experimental/check_shapes/error_contexts.py

@@ -50,7 +50,7 @@
 
 if TYPE_CHECKING:  # pragma: no cover
     from .argument_ref import ArgumentRef
-    from .bools import ParsedBoolSpec
+    from .bool_specs import ParsedBoolSpec
     from .specs import ParsedNoteSpec, ParsedShapeSpec, ParsedTensorSpec
 
 _UNKNOWN_FILE = "<Unknown file>"

tests/integration/test_docs.py

@@ -0,0 +1,45 @@
+# Copyright 2022 the GPflow authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import subprocess
+from pathlib import Path
+
+import pytest
+
+BUILD_DOCS_PATH = Path(__file__).parent.parent.parent / "doc" / "build_docs.py"
+
+
+@pytest.mark.docs
+def test_docs(tmp_path: Path) -> None:
+    assert BUILD_DOCS_PATH.is_file()
+    build_path = tmp_path / "build"
+
+    try:
+        subprocess.run(
+            [
+                "python",
+                str(BUILD_DOCS_PATH),
+                "develop",
+                str(build_path),
+                "--max-notebooks",
+                "0",
+                "--fail-on-warning",
+            ]
+        ).check_returncode()
+    except subprocess.CalledProcessError as e:
+        raise AssertionError(
+            "Documentation build had errors / warnings."
+            " Please fix."
+            " Check both any .rst files you may have modified directly,"
+            " and docstrings of your Python code."
+        ) from e