Merge develop into b/dx-292/row-condition-should-allow-whitespace

ci/azure-pipelines-dev.yml

@@ -164,7 +164,7 @@ stages:
     - job: public_api_report
       steps:
         - script: |
-            python scripts/public_api_report.py
+            python docs/sphinx_api_docs_source/public_api_report.py
           name: PublicAPIReport
 
   - stage: import_ge

ci/azure-pipelines-docs-integration.yml

@@ -89,7 +89,7 @@ stages:
       condition: or(eq(stageDependencies.scope_check.changes.outputs['CheckDocsChanges.DocsChanged'], true), eq(variables.isMain, true))
       steps:
         - script: |
-            python scripts/public_api_report.py
+            python docs/sphinx_api_docs_source/public_api_report.py
           name: PublicAPIReport
 
   - stage: docusaurus_tests

ci/azure-pipelines.yml

@@ -129,7 +129,7 @@ stages:
       condition: or(eq(variables.isScheduled, true), eq(variables.isReleasePrep, true), eq(variables.isRelease, true), eq(variables.isManual, true))
       steps:
         - script: |
-            python scripts/public_api_report.py
+            python docs/sphinx_api_docs_source/public_api_report.py
           name: PublicAPIReport
 
   - stage: docker_tests

docs/build_docs

@@ -19,17 +19,17 @@ then
   git checkout "$GX_LATEST"
   git pull
 else
-  echo -e "${ORANGE}In a pull request, using the latest commit to build API docs so changes can be viewed in the Netlify deploy preview.${NC}"
+  echo -e "${ORANGE}Building from within a pull request, using the latest commit to build API docs so changes can be viewed in the Netlify deploy preview.${NC}"
 fi
 
-echo -e "${ORANGE}Installing dev dependencies.${NC}"
+echo -e "${ORANGE}Installing Great Expectations library dev dependencies.${NC}"
 (cd ../../; pip install -c constraints-dev.txt -e ".[test]")
 
-echo -e "${ORANGE}Installing dev dependencies.${NC}"
+echo -e "${ORANGE}Installing api docs dependencies.${NC}"
 (cd ../sphinx_api_docs_source; pip install -r requirements-dev-api-docs.txt)
 
-echo -e "${ORANGE}Building sphinx API docs.${NC}"
-(cd ../sphinx_api_docs_source; invoke docs)
+echo -e "${ORANGE}Building API docs.${NC}"
+(cd ../../; invoke docs)
 
 if [ "$PULL_REQUEST" == "false" ]
 then
@@ -42,5 +42,5 @@ echo -e "${ORANGE}Copying previous versions${NC}"
 curl "https://superconductive-public.s3.us-east-2.amazonaws.com/oss_docs_versions.zip" -o "oss_docs_versions.zip"
 unzip -oq oss_docs_versions.zip -d .
 
-echo "Building docusaurus docs."
+echo -e "${ORANGE}Building docusaurus docs.${NC}"
 yarn build

docs/docusaurus/README.md

@@ -78,10 +78,10 @@ To add a new version, follow these steps:
 2. Check out the version from the tag, e.g. `git checkout 0.15.50`
 3. Make sure dev dependencies are installed `pip install -c constraints-dev.txt -e ".[test]"`
 4. Install API docs dependencies `pip install -r docs/sphinx_api_docs_source/requirements-dev-api-docs.txt`
-5. Build API docs `cd docs/sphinx_api_docs_source; invoke docs`
-6. Run `yarn install`
-7. Run `yarn build`
-8. Create the version e.g. `yarn docusaurus docs:version 0.15.50`
+5. Build API docs `invoke docs` from the repo root.
+6. Run `yarn install` from `docs/docusaurus/`.
+7. Run `yarn build` from `docs/docusaurus/`.
+8. Create the version e.g. `yarn docusaurus docs:version 0.15.50` from `docs/docusaurus/`.
 9. Pull down the version file (see `docs/build_docs` for the file, currently https://superconductive-public.s3.us-east-2.amazonaws.com/oss_docs_versions.zip)
 10. Unzip and add your newly created versioned docs via the following:
 11. Copy the version you built in step 4 from inside `versioned_docs` in your repo to the `versioned_docs` from the unzipped version file.

docs/sphinx_api_docs_source/build_sphinx_api_docs.py

@@ -30,13 +30,15 @@ def my_task(
 from urllib.parse import urlparse
 
 import invoke
-from bs4 import BeautifulSoup
 
-from scripts.check_public_api_docstrings import (
+from docs.sphinx_api_docs_source.check_public_api_docstrings import (
     get_public_api_definitions,
     get_public_api_module_level_function_definitions,
 )
-from scripts.public_api_report import Definition, get_shortest_dotted_path
+from docs.sphinx_api_docs_source.public_api_report import (
+    Definition,
+    get_shortest_dotted_path,
+)
 
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.StreamHandler())
@@ -65,17 +67,23 @@ class SidebarEntry:
 class SphinxInvokeDocsBuilder:
     """Utility class to support building API docs using Sphinx and Invoke."""
 
-    def __init__(self, ctx: invoke.context.Context, base_path: pathlib.Path) -> None:
+    def __init__(
+        self,
+        ctx: invoke.context.Context,
+        api_docs_source_path: pathlib.Path,
+        repo_root: pathlib.Path,
+    ) -> None:
         """Creates SphinxInvokeDocsBuilder instance.
 
         Args:
             ctx: Invoke context for use in running commands.
-            base_path: Path command is run in for use in determining relative paths.
+            api_docs_source_path: Path to the api docs source. Where sphinx-build is run, should include conf.py.
+            repo_root: Path to the repo root.
         """
         self.ctx = ctx
-        self.base_path = base_path
-        self.docs_path = base_path.parent
-        self.repo_root = self.docs_path.parent
+        self.api_docs_source_path = api_docs_source_path
+        self.docs_path = api_docs_source_path.parent
+        self.repo_root = repo_root
         self.gx_path = self.repo_root / "great_expectations"
 
         self.temp_sphinx_html_dir = self.repo_root / "temp_sphinx_api_docs_build_dir"
@@ -110,11 +118,10 @@ def build_docs(self) -> None:
 
         self._remove_temp_html()
 
-    @staticmethod
-    def exit_with_error_if_docs_dependencies_are_not_installed() -> None:
+    def exit_with_error_if_docs_dependencies_are_not_installed(self) -> None:
         """Checks and report which dependencies are not installed."""
 
-        module_dependencies = ("sphinx", "myst_parser", "pydata_sphinx_theme")
+        module_dependencies = ("sphinx", "myst_parser", "pydata_sphinx_theme", "bs4")
         modules_not_installed = []
 
         for module_name in module_dependencies:
@@ -125,20 +132,19 @@ def exit_with_error_if_docs_dependencies_are_not_installed() -> None:
 
         if modules_not_installed:
             raise invoke.Exit(
-                f"Please make sure to install missing docs dependencies: {', '.join(modules_not_installed)} by running pip install -r docs/sphinx_api_docs_source/requirements-dev-api-docs.txt",
+                f"Please make sure to install missing docs dependencies: {', '.join(modules_not_installed)} by running pip install -r {self.api_docs_source_path / 'requirements-dev-api-docs.txt'}",
                 code=1,
             )
-
         logger.debug("Dependencies installed, proceeding.")
 
     def _build_html_api_docs_in_temp_folder(self):
         """Builds html api documentation in temporary folder."""
 
-        sphinx_api_docs_source_dir = pathlib.Path.cwd()
+        sphinx_api_docs_source_dir = self.api_docs_source_path
         if sphinx_api_docs_source_dir not in sys.path:
             sys.path.append(str(sphinx_api_docs_source_dir))
 
-        cmd = f"sphinx-build -M html ./ {self.temp_sphinx_html_dir} -E"
+        cmd = f"sphinx-build -M html {self.api_docs_source_path} {self.temp_sphinx_html_dir} -E"
         self.ctx.run(cmd, echo=True, pty=True)
         logger.debug("Raw Sphinx HTML generated.")
 
@@ -202,6 +208,10 @@ def _parse_and_process_html_to_mdx(
         Returns:
             Content suitable for use in a docusaurus mdx file.
         """
+        from bs4 import (
+            BeautifulSoup,
+        )  # Importing here since it is not a library requirement
+
         soup = BeautifulSoup(html_file_contents, "html.parser")
 
         # Retrieve and remove the title (it will also be autogenerated by docusaurus)
@@ -403,7 +413,7 @@ def _build_class_md_stubs(self) -> None:
 
     def _write_stub(self, stub: str, path: pathlib.Path) -> None:
         """Write the markdown stub file with appropriate filename."""
-        filepath = self.base_path / path
+        filepath = self.api_docs_source_path / path
         filepath.parent.mkdir(parents=True, exist_ok=True)
         with filepath.open("w") as f:
             f.write(stub)
@@ -535,11 +545,11 @@ def _remove_md_stubs(self):
         """Remove all markdown stub files."""
 
         excluded_files: tuple[pathlib.Path, ...] = (
-            self.base_path / "index.md",
-            self.base_path / "README.md",
+            self.api_docs_source_path / "index.md",
+            self.api_docs_source_path / "README.md",
         )
 
-        all_files: tuple[pathlib.Path] = tuple(self.base_path.glob("*.md"))
+        all_files: tuple[pathlib.Path] = tuple(self.api_docs_source_path.glob("*.md"))
 
         for file in all_files:
             if file not in excluded_files:

scripts/check_public_api_docstrings.py → docs/sphinx_api_docs_source/check_public_api_docstrings.py

@@ -16,7 +16,7 @@
 from dataclasses import dataclass
 from typing import List, Set, Tuple
 
-from scripts.public_api_report import (
+from .public_api_report import (
     CodeParser,
     Definition,
     FileContents,
@@ -89,7 +89,8 @@ def parse_ruff_errors(raw_errors: List[str]) -> List[DocstringError]:
 
 
 def _repo_root() -> pathlib.Path:
-    return pathlib.Path(__file__).parent.parent
+    repo_root_path = pathlib.Path(__file__).parents[2]
+    return repo_root_path
 
 
 def _repo_relative_filepath(filepath: pathlib.Path) -> pathlib.Path:

scripts/public_api_report.py → docs/sphinx_api_docs_source/public_api_report.py

@@ -1764,7 +1764,8 @@ def _deduplicate_strings(self, strings: List[str]) -> List[str]:
 
 
 def _repo_root() -> pathlib.Path:
-    return pathlib.Path(__file__).parent.parent
+    repo_root_path = pathlib.Path(__file__).parents[2]
+    return repo_root_path
 
 
 def _default_doc_example_absolute_paths() -> Set[pathlib.Path]:

tasks.py

@@ -14,13 +14,13 @@
 import os
 import pathlib
 import shutil
-import sys
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Union
 
 import invoke
 from typing_extensions import Final
 
-from scripts import check_public_api_docstrings
+from docs.sphinx_api_docs_source import check_public_api_docstrings, public_api_report
+from docs.sphinx_api_docs_source.build_sphinx_api_docs import SphinxInvokeDocsBuilder
 
 try:
     from tests.integration.usage_statistics import usage_stats_utils
@@ -182,7 +182,7 @@ def hooks(
         ctx.run(" ".join(["pre-commit", "install"]), echo=True)
 
 
-@invoke.task(aliases=["docstring"], iterable=("paths",))
+@invoke.task(aliases=("docstring",), iterable=("paths",))
 def docstrings(ctx: Context, paths: list[str] | None = None):
     """
     Check public API docstrings.
@@ -191,8 +191,6 @@ def docstrings(ctx: Context, paths: list[str] | None = None):
     To pass multiple items:
         invoke docstrings -p=great_expectations/core -p=great_expectations/util.py
     """
-    scripts_path = pathlib.Path.cwd().parent.parent / "scripts"
-    sys.path.append(str(scripts_path))
 
     if paths:
         select_paths = [pathlib.Path(p) for p in paths]
@@ -564,3 +562,49 @@ def _exit_with_error_if_not_in_repo_root(task_name: str):
             exit_message,
             code=1,
         )
+
+
+@invoke.task
+def docs(ctx):
+    """Build documentation. Note: Currently only builds the sphinx based api docs, please build docusaurus docs separately."""
+
+    repo_root = pathlib.Path(__file__).parent
+
+    _exit_with_error_if_not_run_from_correct_dir(
+        task_name="docs", correct_dir=repo_root
+    )
+    sphinx_api_docs_source_dir = repo_root / "docs" / "sphinx_api_docs_source"
+
+    doc_builder = SphinxInvokeDocsBuilder(
+        ctx=ctx, api_docs_source_path=sphinx_api_docs_source_dir, repo_root=repo_root
+    )
+
+    doc_builder.build_docs()
+
+
+@invoke.task(name="public-api")
+def public_api_task(ctx):
+    """Generate a report to determine the state of our Public API. Lists classes, methods and functions that are used in examples in our documentation, and any manual includes or excludes (see public_api_report.py). Items listed when generating this report need the @public_api decorator (and a good docstring) or to be excluded from consideration if they are not applicable to our Public API."""
+
+    repo_root = pathlib.Path(__file__).parent
+
+    _exit_with_error_if_not_run_from_correct_dir(
+        task_name="public-api", correct_dir=repo_root
+    )
+
+    public_api_report.main()
+
+
+def _exit_with_error_if_not_run_from_correct_dir(
+    task_name: str, correct_dir: Union[pathlib.Path, None] = None
+) -> None:
+    """Exit if the command was not run from the correct directory."""
+    if not correct_dir:
+        correct_dir = pathlib.Path(__file__).parent
+    curdir = pathlib.Path.cwd()
+    exit_message = f"The {task_name} task must be invoked from the same directory as the tasks.py file."
+    if correct_dir != curdir:
+        raise invoke.Exit(
+            exit_message,
+            code=1,
+        )

tests/scripts/test_public_api_report.py

@@ -3,8 +3,7 @@
 from typing import List, Union
 
 import pytest
-
-from scripts.public_api_report import (
+from docs.sphinx_api_docs_source.public_api_report import (
     CodeParser,
     CodeReferenceFilter,
     Definition,