Setup docs infra for making crag changes (#4533)

* 1/ copy-n-paste content and code

* 2/ toggle IS_CRAG

* 3/ update content, code, and nav and validate changes

* 4/ enable tests for crag content and code

.buildkite/dagster-buildkite/dagster_buildkite/module_build_spec.py

@@ -12,6 +12,7 @@
     "python_modules/libraries/dagster-docker",
     "python_modules/libraries/lakehouse",
     "examples/docs_snippets",
+    "examples/docs_snippets_crag",
 ]
 
 

.buildkite/dagster-buildkite/dagster_buildkite/steps/dagster.py

@@ -91,6 +91,22 @@ def docs_snippets_extra_cmds_fn(_):
     ]
 
 
+def docs_snippets_crag_extra_cmds_fn(_):
+    return [
+        "pushd examples/docs_snippets_crag",
+        # Run the postgres db. We are in docker running docker
+        # so this will be a sibling container.
+        "docker-compose up -d --remove-orphans",  # clean up in hooks/pre-exit
+        # Can't use host networking on buildkite and communicate via localhost
+        # between these sibling containers, so pass along the ip.
+        network_buildkite_container("postgres"),
+        connect_sibling_docker_container(
+            "postgres", "test-postgres-db-docs-snippets", "POSTGRES_TEST_DB_HOST"
+        ),
+        "popd",
+    ]
+
+
 def deploy_docker_example_extra_cmds_fn(_):
     return [
         "pushd examples/deploy_docker/from_source",
@@ -289,6 +305,13 @@ def graphql_pg_extra_cmds_fn(_):
         upload_coverage=False,
         supported_pythons=ExamplePythons,
     ),
+    ModuleBuildSpec(
+        "examples/docs_snippets_crag",
+        extra_cmds_fn=docs_snippets_crag_extra_cmds_fn,
+        buildkite_label="docs_snippets_crag",
+        upload_coverage=False,
+        supported_pythons=ExamplePythons,
+    ),
     ModuleBuildSpec(
         "examples/hacker_news",
         env_vars=["SNOWFLAKE_ACCOUNT", "SNOWFLAKE_USER", "SNOWFLAKE_PASSWORD"],
@@ -453,6 +476,7 @@ def examples_tests():
     skip_examples = [
         # Skip these folders because they need custom build config
         "docs_snippets",
+        "docs_snippets_crag",
         "airline_demo",
         "dbt_example",
         "deploy_docker",

.buildkite/dagster-buildkite/dagster_buildkite/steps/docs.py

@@ -26,6 +26,16 @@ def docs_steps() -> List[dict]:
         )
         .on_integration_image(SupportedPython.V3_7)
         .build(),
+        # Make sure the docs site can build end-to-end.
+        StepBuilder("crag docs next")
+        .run(
+            "pushd docs/next",
+            "yarn",
+            "yarn crag-test",
+            "yarn crag-build-master",
+        )
+        .on_integration_image(SupportedPython.V3_7)
+        .build(),
         # TODO: Yuhan to fix
         # StepBuilder("docs sphinx json build")
         # .run(

.gitignore

@@ -141,6 +141,7 @@ dask-worker-space
 
 # Examples outputs
 examples/docs_snippets/docs_snippets/**/**/output/
+examples/docs_snippets_crag/docs_snippets_crag/**/**/output/
 examples/**/**/example.db
 
 # Telemetry instance id

Makefile

@@ -13,8 +13,9 @@ check_black:
 	black examples/docs_snippets/docs_snippets/intro_tutorial --check --line-length 78 --target-version py36 --target-version py37 --target-version py38 --fast --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|snapshots/"
 
 isort:
-	isort `git ls-files '*.py' ':!:examples/docs_snippets/docs_snippets/intro_tutorial'`
+	isort `git ls-files '*.py' ':!:examples/docs_snippets/docs_snippets/intro_tutorial' ':!:examples/docs_snippets_crag/docs_snippets_crag/intro_tutorial/*.py'`
 	isort -l 78 `git ls-files 'examples/docs_snippets/docs_snippets/intro_tutorial/*.py'`
+	isort -l 78 `git ls-files 'examples/docs_snippets_crag/docs_snippets_crag/intro_tutorial/*.py'`
 
 yamllint:
 	yamllint -c .yamllint.yaml --strict `git ls-files 'helm/**/*.yml' 'helm/**/*.yaml' ':!:helm/**/templates/*.yml' ':!:helm/**/templates/*.yaml'`

docs/Makefile

@@ -8,7 +8,7 @@ build:
 	cd sphinx; make clean; make json; cd ..; python pack_json.py
 
 snapshot:
-	cd next; yarn snapshot; cd ..;
+	cd next; yarn snapshot && yarn crag-snapshot; cd ..;
 
 update_version:
 	echo "Saving version $(version)"

docs/content-crag/_apidocs.mdx

@@ -0,0 +1,113 @@
+---
+title: API Docs | Dagster
+---
+
+# API Docs
+
+These docs aim to cover the entire public surface of the core `dagster` APIs, as well as public APIs from all provided libraries.
+
+Dagster follows [SemVer](https://semver.org/). We attempt to isolate breaking changes to the public APIs to minor versions (on a roughly 12-week cadence) and will announce deprecations in Slack and in the release notes to patch versions (on a roughly weekly cadence).
+
+## Core
+
+APIs from the core `dagster` package, divided roughly by topic:
+
+- [Solids](/\_apidocs/solids) APIs to define or decorate functions as solids, declare their inputs and outputs, compose solids with each other, as well as the datatypes that solid execution can return or yield.
+
+- [Pipelines](/\_apidocs/pipeline) APIs to define pipelines, dependencies and fan-in dependencies between solids, and aliased instances of solids.
+
+- [Modes & Resources](/\_apidocs/modes-resources) APIs to define pipeline modes and resources.
+
+- [Presets](/\_apidocs/presets) APIs to define configuration presets.
+
+- [Loggers](/\_apidocs/loggers) APIs to define where logs go.
+
+- [Repositories](/\_apidocs/repositories) APIs to define collections of pipelines and other definitions that tools such as the Dagster CLI or Dagit can load from.
+
+- [Config](/\_apidocs/config) The types available to describe config schemas.
+
+- [Types](/\_apidocs/types) Primitive types available for the input and output values of solids, and the APIs used to define and test new Dagster types.
+
+- [Dagster CLI](/\_apidocs/cli) Browse repositories and execute pipelines from the command line
+
+- [Schedules & Sensors](/\_apidocs/schedules-sensors) APIs to define schedules and sensors that initiate pipeline execution, as well as some built-in helpers for common cases.
+
+- [Partitions](/\_apidocs/partitions) APIs to define partitions of the config space over which pipeline runs can be backfilled.
+
+- [Errors](/\_apidocs/errors) Errors thrown by the Dagster framework.
+
+- [Execution](/\_apidocs/execution) APIs to execute and test pipelines and individual solids, the execution context available to solids, pipeline configuration, and the default executors available for executing pipelines.
+
+- [Hooks](/\_apidocs/hooks) APIs to define Dagster hooks, which can be triggered on specific Dagster events.
+
+- [IO Managers](/\_apidocs/io-managers) APIs to define how inputs and outputs are handled and loaded.
+
+- [Dynamic Mapping & Collect](/\_apidocs/dynamic) APIs that allow graph structures to be determined at run time.
+
+- [Experimental Core APIs](/\_apidocs/experimental) The next iteration of Dagster's core abstractions, currently under development.
+
+- [Utilities](/\_apidocs/utilities) Miscellaneous helpers used by Dagster that may be useful to users.
+
+- [Internals](/\_apidocs/internals) Core internal APIs that are important if you are interested in understanding how Dagster works with an eye towards extending it: logging, executors, system storage, the Dagster instance & plugin machinery, storage, schedulers.
+
+## Libraries
+
+Dagster also provides a growing set of optional add-on libraries to integrate with infrastructure and other components of the data ecosystem:
+
+- [Airflow](/\_apidocs/libraries/dagster-airflow) (`dagster-airflow`) Tools for compiling Dagster pipelines to Airflow DAGs, and for ingesting Airflow DAGs to represent them in Dagster.
+
+- [AWS](/\_apidocs/libraries/dagster-aws) (`dagster-aws`) Dagster integrations for working with AWS resources.
+
+- [Azure](/\_apidocs/libraries/dagster-azure) (`dagster-azure`) Dagster integrations for working with Microsoft Azure resources.
+
+- [Celery](/\_apidocs/libraries/dagster-celery) (`dagster-celery`) Provides an executor built on top of the popular [Celery task queue](http://www.celeryproject.org/), and an executor with support for using Celery on Kubernetes.
+
+- [Celery+Docker](/\_apidocs/libraries/dagster-celery-docker) (`dagster-celery-docker`) Provides an executor that lets Celery workers execute in Docker containers.
+
+- [Celery+Kubernetes](/\_apidocs/libraries/dagster-celery-k8s) (`dagster-celery-k8s`) Provides an executor that lets Celery workers execute on Kubernetes.
+
+- [Dask](/\_apidocs/libraries/dagster-dask) (`dagster-dask`) Provides an executor built on top of [dask.distributed](https://distributed.dask.org/en/latest/).
+
+- [dbt](/\_apidocs/libraries/dagster-dbt) (`dagster-dbt`) Provides solids and resources to run [dbt](https://getdbt.com/) projects.
+
+- [Databricks](/\_apidocs/libraries/dagster-databricks) (`dagster-databricks`) Provides solids and resources for integrating with Databricks.
+
+- [Datadog](/\_apidocs/libraries/dagster-datadog) (`dagster-datadog`) Provides an integration with Datadog, to support publishing metrics to Datadog from within Dagster solids.
+
+- [Docker](/\_apidocs/libraries/dagster-docker) (`dagster-docker`) Provides components for deploying Dagster to Docker.
+
+- [GCP](/\_apidocs/libraries/dagster-gcp) (`dagster-gcp`) Dagster integrations for working with Google Cloud Platform resources.
+
+- [GE](/\_apidocs/libraries/dagster-ge) (`dagster-ge`) Dagster integrations for working with [Great Expectations](https://greatexpectations.io/) data quality tests.
+
+- [GitHub](/\_apidocs/libraries/dagster-github) (`dagster-github`) Provides a resource for issuing GitHub GraphQL queries and filing GitHub issues from Dagster pipelines.
+
+- [Kubernetes](/\_apidocs/libraries/dagster-k8s) (`dagster-k8s`) Provides components for deploying Dagster to Kubernetes.
+
+- [MySQL](/\_apidocs/libraries/dagster-mysql) (`dagster-mysql`) Includes implementations of run and event log storage built on MySQL.
+
+- [PagerDuty](/\_apidocs/libraries/dagster-pagerduty) (`dagster-pagerduty`) Provides an integration for generating PagerDuty events from Dagster solids.
+
+- [Pandas](/\_apidocs/libraries/dagster-pandas) (`dagster-pandas`) Provides support for using pandas DataFrames in Dagster and utilities for performing data validation.
+
+- [Papertrail](/\_apidocs/libraries/dagster-papertrail) (`dagster-papertrail`) Provides support for sending Dagster logs to Papertrail.
+
+- [PostgreSQL](/\_apidocs/libraries/dagster-postgres) (`dagster-postgres`) Includes implementations of run and event log storage built on Postgres.
+
+- [Prometheus](/\_apidocs/libraries/dagster-prometheus) (`dagster-prometheus`) Provides support for sending metrics to Prometheus.
+
+- [Pyspark](/\_apidocs/libraries/dagster-pyspark) (`dagster-pyspark`) Provides an integration with pyspark.
+
+- [Shell](/\_apidocs/libraries/dagster-shell) (`dagster-shell`) Provides utilities for issuing shell commands from Dagster pipelines.
+
+- [Slack](/\_apidocs/libraries/dagster-slack) (`dagster-slack`) Provides a simple integration with Slack.
+
+- [Snowflake](/\_apidocs/libraries/dagster-snowflake) (`dagster-snowflake`) Provides a resource for querying Snowflake from Dagster.
+
+- [Spark](/\_apidocs/libraries/dagster-spark) (`dagster-spark`) Provides an integration for working with Spark in Dagster.
+
+- [SSH / SFTP](/\_apidocs/libraries/dagster-ssh) (`dagster-ssh`) Provides an integration for running commands over SSH and retrieving / posting files via SFTP.
+
+- [Twilio](/\_apidocs/libraries/dagster-twilio) (`dagster-twilio`) Provides a resource for posting SMS messages from solids via Twilio.
+
+- [GraphQL](/\_apidocs/libraries/dagster-graphql) (`dagster-graphql`) Provides resources for interfacing with a Dagster deployment over GraphQL.