docs: initial doc generation using Sphinx

.bazelci/presubmit.yml

@@ -23,7 +23,7 @@ buildifier:
   # NOTE: Keep in sync with //:version.bzl
   bazel: 5.4.0
 .minimum_supported_bzlmod_version: &minimum_supported_bzlmod_version
-  bazel: 6.0.0 # test minimum supported version of bazel for bzlmod tests
+  bazel: 6.2.0 # test minimum supported version of bazel for bzlmod tests
 .reusable_config: &reusable_config
   build_targets:
     - "--"
@@ -154,8 +154,9 @@ tasks:
       # on Bazel 5.4 and earlier. To workaround this, manually specify the
       # build kite cc toolchain.
       - "--extra_toolchains=@buildkite_config//config:cc-toolchain"
+      - "--build_tag_filters=-docs"
     test_flags:
-      - "--test_tag_filters=-integration-test,-acceptance-test"
+      - "--test_tag_filters=-integration-test,-acceptance-test,-docs"
       # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1,
       # which prevents cc toolchain autodetection from working correctly
       # on Bazel 5.4 and earlier. To workaround this, manually specify the

.bazelversion

@@ -1 +1 @@
-6.0.0
+6.2.0

.readthedocs.yml

@@ -0,0 +1,10 @@
+
+version: 2
+
+build:
+  os: "ubuntu-22.04"
+  tools:
+    nodejs: "19"
+  commands:
+    - npm install -g @bazel/bazelisk
+    - bazel run //docs/sphinx:readthedocs_install

MODULE.bazel

@@ -54,3 +54,16 @@ use_repo(python, "pythons_hub")
 
 # This call registers the Python toolchains.
 register_toolchains("@pythons_hub//:all")
+
+# ===== DEV ONLY SETUP =====
+docs_pip = use_extension(
+    "//python/extensions:pip.bzl",
+    "pip",
+    dev_dependency = True,
+)
+docs_pip.parse(
+    hub_name = "docs_deps",
+    python_version = "3.11",
+    requirements_darwin = "//docs/sphinx:requirements_darwin.txt",
+    requirements_lock = "//docs/sphinx:requirements_linux.txt",
+)

WORKSPACE

@@ -84,11 +84,13 @@ load("@rules_python_gazelle_plugin//:deps.bzl", _py_gazelle_deps = "gazelle_deps
 # for python requirements.
 _py_gazelle_deps()
 
+# This interpreter is used for various rules_python dev-time tools
+load("@python//3.11.6:defs.bzl", "interpreter")
+
 #####################
 # Install twine for our own runfiles wheel publishing.
 # Eventually we might want to install twine automatically for users too, see:
 # https://github.com/bazelbuild/rules_python/issues/1016.
-load("@python//3.11.6:defs.bzl", "interpreter")
 load("@rules_python//python:pip.bzl", "pip_parse")
 
 pip_parse(
@@ -103,6 +105,21 @@ load("@publish_deps//:requirements.bzl", "install_deps")
 
 install_deps()
 
+#####################
+# Install sphinx for doc generation.
+
+pip_parse(
+    name = "docs_deps",
+    incompatible_generate_aliases = True,
+    python_interpreter_target = interpreter,
+    requirements_darwin = "//docs/sphinx:requirements_darwin.txt",
+    requirements_lock = "//docs/sphinx:requirements_linux.txt",
+)
+
+load("@docs_deps//:requirements.bzl", docs_install_deps = "install_deps")
+
+docs_install_deps()
+
 # This wheel is purely here to validate the wheel extraction code. It's not
 # intended for anything else.
 http_file(

docs/sphinx/BUILD.bazel

@@ -0,0 +1,99 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@docs_deps//:requirements.bzl", "requirement")
+load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+load("//sphinxdocs:readthedocs.bzl", "readthedocs_install")
+load("//sphinxdocs:sphinx.bzl", "sphinx_build_binary", "sphinx_docs")
+load("//sphinxdocs:sphinx_stardoc.bzl", "sphinx_stardocs")
+
+# We only build for Linux and Mac because the actual doc process only runs
+# on Linux. Mac is close enough. Making CI happy under Windows is too much
+# of a headache, though, so we don't bother with that.
+_TARGET_COMPATIBLE_WITH = select({
+    "@platforms//os:linux": [],
+    "@platforms//os:macos": [],
+    "//conditions:default": ["@platforms//:incompatible"],
+})
+
+# See README.md for instructions. Short version:
+# * `bazel run //docs/sphinx:docs.serve` in a separate terminal
+# * `ibazel build //docs/sphinx:docs` to automatically rebuild docs
+sphinx_docs(
+    name = "docs",
+    srcs = [
+        ":bzl_api_docs",
+    ] + glob(
+        include = [
+            "*.md",
+            "**/*.md",
+            "_static/**",
+        ],
+        exclude = ["README.md"],
+    ),
+    config = "conf.py",
+    # Building produces lots of warnings right now because the docs aren't
+    # entirely ready yet. Silence these to reduce the spam in CI logs.
+    extra_opts = ["-Q"],
+    formats = [
+        "html",
+    ],
+    sphinx = ":sphinx-build",
+    strip_prefix = package_name() + "/",
+    tags = ["docs"],
+    target_compatible_with = _TARGET_COMPATIBLE_WITH,
+)
+
+sphinx_stardocs(
+    name = "bzl_api_docs",
+    docs = {
+        "api/cc/py_cc_toolchain.md": dict(
+            dep = "//python/private:py_cc_toolchain_bzl",
+            input = "//python/private:py_cc_toolchain_rule.bzl",
+        ),
+        "api/cc/py_cc_toolchain_info.md": "//python/cc:py_cc_toolchain_info_bzl",
+        "api/defs.md": "//python:defs_bzl",
+        "api/entry_points/py_console_script_binary.md": "//python/entry_points:py_console_script_binary_bzl",
+        "api/packaging.md": "//python:packaging_bzl",
+        "api/pip.md": "//python:pip_bzl",
+    },
+    tags = ["docs"],
+    target_compatible_with = _TARGET_COMPATIBLE_WITH,
+)
+
+readthedocs_install(
+    name = "readthedocs_install",
+    docs = [":docs"],
+    target_compatible_with = _TARGET_COMPATIBLE_WITH,
+)
+
+sphinx_build_binary(
+    name = "sphinx-build",
+    target_compatible_with = _TARGET_COMPATIBLE_WITH,
+    deps = [
+        requirement("sphinx"),
+        requirement("sphinx_rtd_theme"),
+        requirement("myst_parser"),
+        requirement("readthedocs_sphinx_ext"),
+    ],
+)
+
+# Run bazel run //docs/sphinx:requirements.update
+compile_pip_requirements(
+    name = "requirements",
+    requirements_darwin = "requirements_darwin.txt",
+    requirements_in = "requirements.in",
+    requirements_txt = "requirements_linux.txt",
+    target_compatible_with = _TARGET_COMPATIBLE_WITH,
+)

docs/sphinx/README.md

@@ -0,0 +1,72 @@
+# rules_python Sphinx docs generation
+
+The docs for rules_python are generated using a combination of Sphinx, Bazel,
+and Readthedocs.org. The Markdown files in source control are unlikely to render
+properly without the Sphinx processing step because they rely on Sphinx and
+MyST-specific Markdown functionality.
+
+The actual sources that Sphinx consumes are in this directory, with Stardoc
+generating additional sources or Sphinx.
+
+Manually building the docs isn't necessary -- readthedocs.org will
+automatically build and deploy them when commits are pushed to the repo.
+
+## Generating docs for development
+
+Generating docs for development is a two-part process: starting a local HTTP
+server to serve the generated HTML, and re-generating the HTML when sources
+change. The quick start is:
+
+```
+bazel run //docs/sphinx:docs.serve  # Run in separate terminal
+ibazel build //docs/sphinx:docs  # Automatically rebuilds docs
+```
+
+This will build the docs and start a local webserver at http://localhost:8000
+where you can view the output. As you edit files, ibazel will detect the file
+changes and re-run the build process, and you can simply refresh your browser to
+see the changes. Using ibazel is not required; you can manually run the
+equivalent bazel command if desired.
+
+### Installing ibazel
+
+The `ibazel` tool can be used to automatically rebuild the docs as you
+development them. See the [ibazel docs](https://github.com/bazelbuild/bazel-watcher) for
+how to install it. The quick start for linux is:
+
+```
+sudo apt install npm
+sudo npm install -g @bazel/ibazel
+```
+
+## MyST Markdown flavor
+
+Sphinx is configured to parse Markdown files using MyST, which is a more
+advanced flavor of Markdown that supports most features of restructured text and
+integrates with Sphinx functionality such as automatic cross references,
+creating indexes, and using concise markup to generate rich documentation.
+
+MyST features and behaviors are controlled by the Sphinx configuration file,
+`docs/sphinx/conf.py`. For more info, see https://myst-parser.readthedocs.io.
+
+## Sphinx configuration
+
+The Sphinx-specific configuration files and input doc files live in
+docs/sphinx.
+
+The Sphinx configuration is `docs/sphinx/conf.py`. See
+https://www.sphinx-doc.org/ for details about the configuration file.
+
+## Readthedocs configuration
+
+There's two basic parts to the readthedocs configuration:
+
+*   `.readthedocs.yaml`: This configuration file controls most settings, such as
+    the OS version used to build, Python version, dependencies, what Bazel
+    commands to run, etc.
+*   https://readthedocs.org/projects/rules-python: This is the project
+    administration page. While most settings come from the config file, this
+    controls additional settings such as permissions, what versions are
+    published, when to publish changes, etc.
+
+For more readthedocs configuration details, see docs.readthedocs.io.

docs/sphinx/_static/css/custom.css

@@ -0,0 +1,34 @@
+.wy-nav-content {
+  max-width: 70%;
+}
+
+.starlark-object {
+  border: thin solid grey;
+  margin-bottom: 1em;
+}
+
+.starlark-object h2 {
+  background-color: #e7f2fa;
+  border-bottom: thin solid grey;
+  padding-left: 0.5ex;
+}
+
+.starlark-object>p, .starlark-object>dl {
+  /* Prevent the words from touching the border line */
+  padding-left: 0.5ex;
+}
+
+.starlark-signature {
+  font-family: monospace;
+}
+
+/* Fixup the headerlinks in param names */
+.starlark-object dt a {
+  /* Offset the link icon to be outside the colon */
+  position: relative;
+  right: -1ex;
+  /* Remove the empty space between the param name and colon */
+  width: 0;
+  /* Override the .headerlink margin */
+  margin-left: 0 !important;
+}

docs/sphinx/api/index.md

@@ -0,0 +1,6 @@
+# API Reference
+
+```{toctree}
+:glob:
+**
+```

docs/sphinx/conf.py

@@ -0,0 +1,73 @@
+# Configuration file for the Sphinx documentation builder.
+
+# -- Project information
+project = "rules_python"
+copyright = "2023, The Bazel Authors"
+author = "Bazel"
+
+# Readthedocs fills these in
+release = "0.0.0"
+version = release
+
+# -- General configuration
+
+# Any extensions here not built into Sphinx must also be added to
+# the dependencies of Bazel and Readthedocs.
+# * //docs:requirements.in
+# * Regenerate //docs:requirements.txt (used by readthedocs)
+# * Add the dependencies to //docs:sphinx_build
+extensions = [
+    "sphinx.ext.duration",
+    "sphinx.ext.doctest",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.autosectionlabel",
+    "myst_parser",
+    "sphinx_rtd_theme",  # Necessary to get jquery to make flyout work
+]
+
+exclude_patterns = ["crossrefs.md"]
+
+intersphinx_mapping = {}
+
+intersphinx_disabled_domains = ["std"]
+
+# Prevent local refs from inadvertently linking elsewhere, per
+# https://docs.readthedocs.io/en/stable/guides/intersphinx.html#using-intersphinx
+intersphinx_disabled_reftypes = ["*"]
+
+templates_path = ["_templates"]
+
+# -- Options for HTML output
+
+html_theme = "sphinx_rtd_theme"
+
+# See https://sphinx-rtd-theme.readthedocs.io/en/stable/configuring.html
+# for options
+html_theme_options = {}
+
+# Keep this in sync with the stardoc templates
+html_permalinks_icon = "¶"
+
+# See https://myst-parser.readthedocs.io/en/latest/syntax/optional.html
+# for additional extensions.
+myst_enable_extensions = [
+    "fieldlist",
+    "attrs_block",
+    "attrs_inline",
+    "colon_fence",
+    "deflist",
+]
+
+# These folders are copied to the documentation's HTML output
+html_static_path = ["_static"]
+
+# These paths are either relative to html_static_path
+# or fully qualified paths (eg. https://...)
+html_css_files = [
+    "css/custom.css",
+]
+
+# -- Options for EPUB output
+epub_show_urls = "footnote"