docs: use stardoc proto output to generate markdown docs

.bazelrc

@@ -25,5 +25,8 @@ startup --windows_enable_symlinks
 common --noexperimental_enable_bzlmod
 
 # Additional config to use for readthedocs builds.
-# See .readthedocs.yml for additional flags
+# See .readthedocs.yml for additional flags that can only be determined from
+# the runtime environment.
 build:rtd --stamp
+# Some bzl files contain repos only available under bzlmod
+build:rtd --enable_bzlmod

CHANGELOG.md

@@ -47,6 +47,10 @@ A brief description of the categories of changes:
 * (bzlmod pip.parse) Requirements files with duplicate entries for the same
   package (e.g. one for the package, one for an extra) now work.
 
+### Added
+
+* (docs) bzlmod extensions are now documented on rules-python.readthedocs.io
+
 [0.XX.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.XX.0
 
 ## [0.27.0] - 2023-11-16

MODULE.bazel

@@ -56,12 +56,12 @@ use_repo(python, "pythons_hub")
 register_toolchains("@pythons_hub//:all")
 
 # ===== DEV ONLY SETUP =====
-docs_pip = use_extension(
+dev_pip = use_extension(
     "//python/extensions:pip.bzl",
     "pip",
     dev_dependency = True,
 )
-docs_pip.parse(
+dev_pip.parse(
     experimental_requirement_cycles = {
         "sphinx": [
             "sphinx",
@@ -72,7 +72,7 @@ docs_pip.parse(
             "sphinxcontrib-applehelp",
         ],
     },
-    hub_name = "docs_deps",
+    hub_name = "dev_pip",
     python_version = "3.11",
     requirements_darwin = "//docs/sphinx:requirements_darwin.txt",
     requirements_lock = "//docs/sphinx:requirements_linux.txt",

WORKSPACE

@@ -109,7 +109,7 @@ install_deps()
 # Install sphinx for doc generation.
 
 pip_parse(
-    name = "docs_deps",
+    name = "dev_pip",
     experimental_requirement_cycles = {
         "sphinx": [
             "sphinx",
@@ -126,7 +126,7 @@ pip_parse(
     requirements_lock = "//docs/sphinx:requirements_linux.txt",
 )
 
-load("@docs_deps//:requirements.bzl", docs_install_deps = "install_deps")
+load("@dev_pip//:requirements.bzl", docs_install_deps = "install_deps")
 
 docs_install_deps()
 
@@ -140,3 +140,9 @@ http_file(
         "https://files.pythonhosted.org/packages/50/67/3e966d99a07d60a21a21d7ec016e9e4c2642a86fea251ec68677daf71d4d/numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
     ],
 )
+
+# rules_proto expects //external:python_headers to point at the python headers.
+bind(
+    name = "python_headers",
+    actual = "//python/cc:current_py_cc_headers",
+)

docs/sphinx/BUILD.bazel

@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("@docs_deps//:requirements.bzl", "requirement")
-load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+load("@dev_pip//:requirements.bzl", "requirement")
+load("//python:pip.bzl", "compile_pip_requirements")
+load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")  # buildifier: disable=bzl-visibility
+load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER")  # buildifier: disable=bzl-visibility
 load("//sphinxdocs:readthedocs.bzl", "readthedocs_install")
 load("//sphinxdocs:sphinx.bzl", "sphinx_build_binary", "sphinx_docs", "sphinx_inventory")
 load("//sphinxdocs:sphinx_stardoc.bzl", "sphinx_stardocs")
@@ -83,7 +85,13 @@ sphinx_stardocs(
         "api/entry_points/py_console_script_binary.md": "//python/entry_points:py_console_script_binary_bzl",
         "api/packaging.md": "//python:packaging_bzl",
         "api/pip.md": "//python:pip_bzl",
-    },
+    } | ({
+        # Bazel 6 + Stardoc isn't able to parse something about the python bzlmod extension
+        "api/extensions/python.md": "//python/extensions:python_bzl",
+    } if IS_BAZEL_7_OR_HIGHER else {}) | ({
+        # This depends on @pythons_hub, which is only created under bzlmod
+        "api/extensions/pip.md": "//python/extensions:pip_bzl",
+    } if BZLMOD_ENABLED else {}),
     footer = "_stardoc_footer.md",
     tags = ["docs"],
     target_compatible_with = _TARGET_COMPATIBLE_WITH,

docs/sphinx/_stardoc_footer.md

@@ -7,6 +7,8 @@
 [`Label`]: https://bazel.build/rules/lib/Label
 [`list`]: https://bazel.build/rules/lib/list
 [`str`]: https://bazel.build/rules/lib/string
+[str]: https://bazel.build/rules/lib/string
+[`int`]: https://bazel.build/rules/lib/int
 [`struct`]: https://bazel.build/rules/lib/builtins/struct
 [`Target`]: https://bazel.build/rules/lib/Target
 [target-name]: https://bazel.build/concepts/labels#target-names

docs/sphinx/_static/css/custom.css

@@ -12,8 +12,17 @@
   border-bottom: thin solid grey;
   padding-left: 0.5ex;
 }
+.starlark-object h3 {
+  background-color: #e7f2fa;
+  padding-left: 0.5ex;
+}
+
+.starlark-module-extension-tag-class h3 {
+  background-color: #add8e6;
+  padding-left: 0.5ex;
+}
 
-.starlark-object>p, .starlark-object>dl {
+.starlark-object>p, .starlark-object>dl, .starlark-object>section>* {
   /* Prevent the words from touching the border line */
   padding-left: 0.5ex;
 }

docs/sphinx/pyproject.toml

@@ -9,4 +9,5 @@ dependencies = [
     "myst-parser",
     "sphinx_rtd_theme",
     "readthedocs-sphinx-ext",
+    "absl-py",
 ]

docs/sphinx/readthedocs_build.sh

@@ -14,6 +14,7 @@ extra_env+=("--//sphinxdocs:extra_env=HOSTNAME=$HOSTNAME")
 
 set -x
 bazel run \
+  --config=rtd \
   "--//sphinxdocs:extra_defines=version=$READTHEDOCS_VERSION" \
   "${extra_env[@]}" \
   //docs/sphinx:readthedocs_install

docs/sphinx/requirements_linux.txt

@@ -4,6 +4,10 @@
 #
 #    bazel run //docs/sphinx:requirements.update
 #
+absl-py==2.0.0 \
+    --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3 \
+    --hash=sha256:d9690211c5fcfefcdd1a45470ac2b5c5acd45241c3af71eed96bc5441746c0d5
+    # via rules-python-docs (docs/sphinx/pyproject.toml)
 alabaster==0.7.13 \
     --hash=sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3 \
     --hash=sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2

python/BUILD.bazel

@@ -196,6 +196,7 @@ bzl_library(
     srcs = ["repositories.bzl"],
     deps = [
         ":versions_bzl",
+        "//python/pip_install:repositories_bzl",
         "//python/private:auth_bzl",
         "//python/private:bazel_tools_bzl",
         "//python/private:bzlmod_enabled_bzl",

python/extensions/BUILD.bazel

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])
@@ -21,3 +23,17 @@ filegroup(
     srcs = glob(["**"]),
     visibility = ["//python:__pkg__"],
 )
+
+bzl_library(
+    name = "pip_bzl",
+    srcs = ["pip.bzl"],
+    visibility = ["//:__subpackages__"],
+    deps = ["//python/private/bzlmod:pip_bzl"],
+)
+
+bzl_library(
+    name = "python_bzl",
+    srcs = ["python.bzl"],
+    visibility = ["//:__subpackages__"],
+    deps = ["//python/private/bzlmod:python_bzl"],
+)

python/pip_install/pip_repository.bzl

@@ -678,7 +678,7 @@ alias(
 )
 ```
 
-## Vendoring the requirements.bzl file
+### Vendoring the requirements.bzl file
 
 In some cases you may not want to generate the requirements.bzl file as a repository rule
 while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module

python/private/bzlmod/BUILD.bazel

@@ -13,8 +13,9 @@
 # limitations under the License.
 
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")
 
-package(default_visibility = ["//visibility:private"])
+package(default_visibility = ["//:__subpackages__"])
 
 licenses(["notice"])
 
@@ -24,6 +25,28 @@ filegroup(
     visibility = ["//python/private:__pkg__"],
 )
 
+bzl_library(
+    name = "pip_bzl",
+    srcs = ["pip.bzl"],
+    deps = [
+        ":pip_repository_bzl",
+        "//python/pip_install:pip_repository_bzl",
+        "//python/pip_install:requirements_parser_bzl",
+        "//python/private:full_version_bzl",
+        "//python/private:normalize_name_bzl",
+        "//python/private:parse_whl_name_bzl",
+        "//python/private:version_label_bzl",
+        ":bazel_features_bzl",
+    ] + [
+        "@pythons_hub//:interpreters_bzl",
+    ] if BZLMOD_ENABLED else [],
+)
+
+bzl_library(
+    name = "bazel_features_bzl",
+    srcs = ["@bazel_features//:bzl_files"] if BZLMOD_ENABLED else [],
+)
+
 bzl_library(
     name = "pip_repository_bzl",
     srcs = ["pip_repository.bzl"],
@@ -33,3 +56,23 @@ bzl_library(
         "//python/private:text_util_bzl",
     ],
 )
+
+bzl_library(
+    name = "python_bzl",
+    srcs = ["python.bzl"],
+    deps = [
+        ":pythons_hub_bzl",
+        "//python:repositories_bzl",
+        "//python/private:toolchains_repo_bzl",
+    ],
+)
+
+bzl_library(
+    name = "pythons_hub_bzl",
+    srcs = ["pythons_hub.bzl"],
+    deps = [
+        "//python:versions_bzl",
+        "//python/private:full_version_bzl",
+        "//python/private:toolchains_repo_bzl",
+    ],
+)

python/private/bzlmod/pythons_hub.bzl

@@ -29,7 +29,19 @@ def _have_same_length(*lists):
         fail("expected at least one list")
     return len({len(length): None for length in lists}) == 1
 
-def _python_toolchain_build_file_content(
+_HUB_BUILD_FILE_TEMPLATE = """\
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+
+bzl_library(
+    name = "interpreters_bzl",
+    srcs = ["interpreters.bzl"],
+    visibility = ["@rules_python//:__subpackages__"],
+)
+
+{toolchains}
+"""
+
+def _hub_build_file_content(
         prefixes,
         python_versions,
         set_python_version_constraints,
@@ -48,15 +60,17 @@ def _python_toolchain_build_file_content(
 
     # Iterate over the length of python_versions and call
     # build the toolchain content by calling python_toolchain_build_file_content
-    return "\n".join([python_toolchain_build_file_content(
+    toolchains = "\n".join([python_toolchain_build_file_content(
         prefix = prefixes[i],
         python_version = full_version(python_versions[i]),
         set_python_version_constraint = set_python_version_constraints[i],
         user_repository_name = user_repository_names[i],
         rules_python = rules_python,
     ) for i in range(len(python_versions))])
 
-_build_file_for_hub_template = """
+    return _HUB_BUILD_FILE_TEMPLATE.format(toolchains = toolchains)
+
+_interpreters_bzl_template = """
 INTERPRETER_LABELS = {{
 {interpreter_labels}
 }}
@@ -72,7 +86,7 @@ def _hub_repo_impl(rctx):
     # write them to the BUILD file.
     rctx.file(
         "BUILD.bazel",
-        _python_toolchain_build_file_content(
+        _hub_build_file_content(
             rctx.attr.toolchain_prefixes,
             rctx.attr.toolchain_python_versions,
             rctx.attr.toolchain_set_python_version_constraints,
@@ -97,7 +111,7 @@ def _hub_repo_impl(rctx):
 
     rctx.file(
         "interpreters.bzl",
-        _build_file_for_hub_template.format(
+        _interpreters_bzl_template.format(
             interpreter_labels = interpreter_labels,
             default_python_version = rctx.attr.default_python_version,
         ),

sphinxdocs/private/BUILD.bazel

@@ -13,7 +13,9 @@
 # limitations under the License.
 
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load("//python:proto.bzl", "py_proto_library")
 load("//python:py_binary.bzl", "py_binary")
+load("//python:py_library.bzl", "py_library")
 
 package(
     default_visibility = ["//sphinxdocs:__subpackages__"],
@@ -70,3 +72,28 @@ py_binary(
     # Only public because it's an implicit attribute
     visibility = ["//:__subpackages__"],
 )
+
+py_binary(
+    name = "proto_to_markdown",
+    srcs = ["proto_to_markdown.py"],
+    # Only public because it's an implicit attribute
+    visibility = ["//:__subpackages__"],
+    deps = [":proto_to_markdown_lib"],
+)
+
+py_library(
+    name = "proto_to_markdown_lib",
+    srcs = ["proto_to_markdown.py"],
+    # Only public because it's an implicit attribute
+    visibility = ["//:__subpackages__"],
+    deps = [
+        ":stardoc_output_proto_py_pb2",
+    ],
+)
+
+py_proto_library(
+    name = "stardoc_output_proto_py_pb2",
+    deps = [
+        "@io_bazel_stardoc//stardoc/proto:stardoc_output_proto",
+    ],
+)