Docs non crag (#5337)

testing:
- make snapshot
- make black && make isort
- make build
- yarn test
- yarn build-master

testing yarn build on vercel

releasing: 
- version=foo make update_version && python remove_version.py --version foo

.arclint

@@ -8,7 +8,7 @@
     "isort": {
       "type": "isort",
       "include": "(\\.py$)",
-      "exclude": "(examples/docs_snippets_crag/docs_snippets_crag/intro_tutorial/)"
+      "exclude": "(examples/docs_snippets/docs_snippets/intro_tutorial/)"
     },
     "eslint": {
       "type": "eslint",

.buildkite/dagster-buildkite/dagster_buildkite/module_build_spec.py

@@ -10,7 +10,7 @@
     "python_modules/libraries/dagster-databricks",
     "python_modules/libraries/dagster-dbt",
     "python_modules/libraries/dagster-docker",
-    "examples/docs_snippets_crag",
+    "examples/docs_snippets",
 ]
 
 

.buildkite/dagster-buildkite/dagster_buildkite/steps/dagster.py

@@ -77,7 +77,7 @@ def dbt_example_extra_cmds_fn(_):
 
 def docs_snippets_extra_cmds_fn(_):
     return [
-        "pushd examples/docs_snippets_crag",
+        "pushd examples/docs_snippets",
         # Run the postgres db. We are in docker running docker
         # so this will be a sibling container.
         "docker-compose up -d --remove-orphans",  # clean up in hooks/pre-exit
@@ -283,7 +283,7 @@ def graphql_pg_extra_cmds_fn(_):
         supported_pythons=ExamplePythons,
     ),
     ModuleBuildSpec(
-        "examples/docs_snippets_crag",
+        "examples/docs_snippets",
         extra_cmds_fn=docs_snippets_extra_cmds_fn,
         buildkite_label="docs_snippets",
         upload_coverage=False,
@@ -458,7 +458,7 @@ def examples_tests():
 
     skip_examples = [
         # Skip these folders because they need custom build config
-        "docs_snippets_crag",
+        "docs_snippets",
         "airline_demo",
         "dbt_example",
         "deploy_docker",

.gitignore

@@ -141,7 +141,7 @@ dask-worker-space
 
 # Examples outputs
 examples/docs_snippets/docs_snippets/**/**/output/
-examples/docs_snippets_crag/docs_snippets_crag/**/**/output/
+examples/docs_snippets/docs_snippets/**/**/output/
 examples/**/**/example.db
 
 # Telemetry instance id

Makefile

@@ -6,15 +6,15 @@ update_doc_snapshot:
 
 black:
 	black examples integration_tests helm python_modules .buildkite --line-length 100 --target-version py36 --target-version py37 --target-version py38 --fast --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|snapshots/|intro_tutorial/"
-	black examples/docs_snippets_crag/docs_snippets_crag/intro_tutorial --line-length 78 --target-version py36 --target-version py37 --target-version py38 --fast --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|snapshots/"
+	black examples/docs_snippets/docs_snippets/intro_tutorial --line-length 78 --target-version py36 --target-version py37 --target-version py38 --fast --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|snapshots/"
 
 check_black:
 	black examples integration_tests helm python_modules .buildkite --check --line-length 100 --target-version py36 --target-version py37 --target-version py38 --fast --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|snapshots/|intro_tutorial/"
-	black examples/docs_snippets_crag/docs_snippets_crag/intro_tutorial --check --line-length 78 --target-version py36 --target-version py37 --target-version py38 --fast --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|snapshots/"
+	black examples/docs_snippets/docs_snippets/intro_tutorial --check --line-length 78 --target-version py36 --target-version py37 --target-version py38 --fast --exclude "build/|buck-out/|dist/|_build/|\.eggs/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|snapshots/"
 
 isort:
-	isort `git ls-files '*.py' ':!:examples/docs_snippets_crag/docs_snippets_crag/intro_tutorial'`
-	isort -l 78 `git ls-files 'examples/docs_snippets_crag/docs_snippets_crag/intro_tutorial/*.py'`
+	isort `git ls-files '*.py' ':!:examples/docs_snippets/docs_snippets/intro_tutorial'`
+	isort -l 78 `git ls-files 'examples/docs_snippets/docs_snippets/intro_tutorial/*.py'`
 
 yamllint:
 	yamllint -c .yamllint.yaml --strict `git ls-files 'helm/**/*.yml' 'helm/**/*.yaml' ':!:helm/**/templates/*.yml' ':!:helm/**/templates/*.yaml'`

docs/content-crag/guides/dagster/re-execution.mdx → docs/content/guides/dagster/re-execution.mdx

@@ -82,7 +82,7 @@ Again, let's revist the job `unreliable_job`, which has a op named `unreliable`.
 
 ```python file=/guides/dagster/reexecution/reexecution_api.py endbefore=end_initial_execution_marker
 from dagster import DagsterInstance, execute_pipeline, reexecute_pipeline
-from docs_snippets_crag.guides.dagster.reexecution.unreliable_job import unreliable_job
+from docs_snippets.guides.dagster.reexecution.unreliable_job import unreliable_job
 
 instance = DagsterInstance.ephemeral()
 

docs/content-crag/integrations/airflow.mdx → docs/content/integrations/airflow.mdx

@@ -65,7 +65,7 @@ DEFAULT_ARGS = {
 }
 
 dag, tasks = make_airflow_dag(
-    module_name="docs_snippets_crag.integrations.airflow.hello_cereal",
+    module_name="docs_snippets.integrations.airflow.hello_cereal",
     pipeline_name="hello_cereal_pipeline",
     dag_kwargs={"default_args": DEFAULT_ARGS, "max_active_runs": 1},
 )
@@ -80,7 +80,7 @@ If you run this code interactively, you'll see that `dag` and `tasks` are ordina
 [<Task(DagsterPythonOperator): hello_cereal>]
 ```
 
-Like other Airflow DAG definition files, this should go inside `$AIRLFLOW_HOME/dags`. The `docs_snippets_crag.integrations.airflow.hello_cereal` module that's passed as the value for the `module_name` argument must be importable via the sys.path.
+Like other Airflow DAG definition files, this should go inside `$AIRLFLOW_HOME/dags`. The `docs_snippets.integrations.airflow.hello_cereal` module that's passed as the value for the `module_name` argument must be importable via the sys.path.
 
 After this, the DAG should show up inside Airflow:
 
@@ -116,7 +116,7 @@ DEFAULT_ARGS = {
 }
 
 dag, steps = make_airflow_dag_containerized(
-    module_name="docs_snippets_crag.integrations.airflow.hello_cereal",
+    module_name="docs_snippets.integrations.airflow.hello_cereal",
     pipeline_name="hello_cereal_pipeline",
     image="dagster-airflow-demo-repository",
     dag_kwargs={"default_args": DEFAULT_ARGS, "max_active_runs": 1},

docs/content-crag/integrations/dagstermill.mdx → docs/content/integrations/dagstermill.mdx

@@ -98,7 +98,7 @@ We'll illustrate this process by adding a non-notebook solid to our pipeline, wh
 import dagstermill as dm
 from dagster import InputDefinition, ModeDefinition, fs_io_manager, pipeline
 from dagster.utils import script_relative_path
-from docs_snippets_crag.legacy.data_science.download_file import download_file
+from docs_snippets.legacy.data_science.download_file import download_file
 
 k_means_iris = dm.define_dagstermill_solid(
     "k_means_iris",
@@ -220,7 +220,7 @@ For instance, suppose we want to make the number of clusters (the \_k\_ in k-mea
 import dagstermill as dm
 from dagster import Field, InputDefinition, Int, ModeDefinition, fs_io_manager, pipeline
 from dagster.utils import script_relative_path
-from docs_snippets_crag.legacy.data_science.download_file import download_file
+from docs_snippets.legacy.data_science.download_file import download_file
 
 k_means_iris = dm.define_dagstermill_solid(
     "k_means_iris",

docs/content-crag/tutorial/advanced-tutorial/configuring-ops.mdx → docs/content/tutorial/advanced-tutorial/configuring-ops.mdx

@@ -5,7 +5,7 @@ description: Ops can be parameterized with runtime configuration
 
 # Configuring Ops
 
-<CodeReferenceLink filePath="examples/docs_snippets_crag/docs_snippets_crag/intro_tutorial/advanced/configuring_ops/" />
+<CodeReferenceLink filePath="examples/docs_snippets/docs_snippets/intro_tutorial/advanced/configuring_ops/" />
 
 In many situations, we'd like to be able to configure ops at run time. For example, we may want whomever is running the job to decide what dataset it operates on. Configuration is useful when you want the person or system that is launching a job to be able to make choices about what the job does, without needing to modify the job definition.
 
@@ -81,7 +81,7 @@ Select the "Launchpad" tab to get to the place for entering config and launching
 ops:
   download_csv:
     config:
-      url: "https://raw.githubusercontent.com/dagster-io/dagster/master/examples/docs_snippets_crag/docs_snippets_crag/intro_tutorial/cereal.csv"
+      url: "https://raw.githubusercontent.com/dagster-io/dagster/master/examples/docs_snippets/docs_snippets/intro_tutorial/cereal.csv"
 ```
 
 Let's enter it into the config editor:
@@ -127,7 +127,7 @@ When executing jobs with the Dagster CLI, we'll need to provide the run config i
 ops:
   download_csv:
     config:
-      url: "https://raw.githubusercontent.com/dagster-io/dagster/master/examples/docs_snippets_crag/docs_snippets_crag/intro_tutorial/cereal.csv"
+      url: "https://raw.githubusercontent.com/dagster-io/dagster/master/examples/docs_snippets/docs_snippets/intro_tutorial/cereal.csv"
 ```
 
 We can pass config files in this format to the Dagster CLI tool with the `-c` flag.

docs/content-crag/tutorial/advanced-tutorial/materializations.mdx → docs/content/tutorial/advanced-tutorial/materializations.mdx

@@ -5,7 +5,7 @@ description: Dagster ops can describe their persistent artifacts to the system b
 
 # Advanced: Materializations
 
-<CodeReferenceLink filePath="examples/docs_snippets_crag/docs_snippets_crag/intro_tutorial/advanced/materializations/" />
+<CodeReferenceLink filePath="examples/docs_snippets/docs_snippets/intro_tutorial/advanced/materializations/" />
 
 Steps in a job often produce persistent artifacts, for instance, graphs or tables describing the result of some computation. Typically these artifacts are saved to disk (or to cloud storage) with a [name](https://xkcd.com/1459/) that has something to do with their origin. But it can be hard to organize and cross-reference artifacts produced by many different runs of a job, or to identify all of the files that might have been created by some job's logic.
 

docs/content-crag/tutorial/advanced-tutorial/repositories.mdx → docs/content/tutorial/advanced-tutorial/repositories.mdx

@@ -5,7 +5,7 @@ description: Dagster provides the concept of a repository, a collection of jobs
 
 # Advanced: Organizing Jobs in Repositories & Workspaces
 
-<CodeReferenceLink filePath="examples/docs_snippets_crag/docs_snippets_crag/intro_tutorial/advanced/repositories/" />
+<CodeReferenceLink filePath="examples/docs_snippets/docs_snippets/intro_tutorial/advanced/repositories/" />
 
 In all of the examples we've seen so far, we've specified a file (`-f`) or a module (`-m`) in order to tell the CLI tools how to load a job, e.g.:
 
@@ -93,7 +93,7 @@ Then, you can configure your `workspace.yaml` to load jobs from this package.
 
 ```yaml file=/intro_tutorial/advanced/repositories/workspace.yaml
 load_from:
-  - python_package: docs_snippets_crag.intro_tutorial.advanced.repositories.repos
+  - python_package: docs_snippets.intro_tutorial.advanced.repositories.repos
 ```
 
 ### Supporting multiple repositories
@@ -102,7 +102,7 @@ You can also use `workspace.yaml` to load multiple repositories. This can be use
 
 ```yaml file=/intro_tutorial/advanced/repositories/multi_repo_workspace.yaml
 load_from:
-  - python_package: docs_snippets_crag.intro_tutorial.advanced.repositories.repos
+  - python_package: docs_snippets.intro_tutorial.advanced.repositories.repos
   - python_file: repos.py
 ```
 

docs/content-crag/tutorial/advanced-tutorial/scheduling.mdx → docs/content/tutorial/advanced-tutorial/scheduling.mdx

@@ -5,7 +5,7 @@ description: Dagster includes a scheduler that can be used to run jobs at regula
 
 # Advanced: Scheduling Job Runs
 
-<CodeReferenceLink filePath="examples/docs_snippets_crag/docs_snippets_crag/intro_tutorial/advanced/scheduling/" />
+<CodeReferenceLink filePath="examples/docs_snippets/docs_snippets/intro_tutorial/advanced/scheduling/" />
 
 Dagster includes a scheduler that allows you to run jobs at regular intervals, e.g. daily or hourly.
 

docs/content-crag/tutorial/advanced-tutorial/types.mdx → docs/content/tutorial/advanced-tutorial/types.mdx

@@ -5,7 +5,7 @@ description: Besides Python 3's typing system, Dagster provides a type system th
 
 # Advanced: Dagster Types
 
-<CodeReferenceLink filePath="examples/docs_snippets_crag/docs_snippets_crag/intro_tutorial/basics/e04_quality/" />
+<CodeReferenceLink filePath="examples/docs_snippets/docs_snippets/intro_tutorial/basics/e04_quality/" />
 
 ## Verifying Op Outputs and Inputs
 

docs/next/__tests__/mdxInternalLinks.test.ts

@@ -4,7 +4,7 @@ import fg from "fast-glob";
 import { Node } from "hast";
 import visit from "unist-util-visit";
 import { flatten } from "../util/useNavigation";
-import masterNavigation from "../../content-crag/_navigation.json";
+import masterNavigation from "../../content/_navigation.json";
 import generateToc from "mdast-util-toc";
 import { getItems, getIds } from "../components/mdx/SidebarNavigation";
 import matter from "gray-matter";
@@ -14,7 +14,7 @@ import mdx from "remark-mdx";
 import remark from "remark";
 
 const ROOT_DIR = path.resolve(__dirname, "../../");
-const DOCS_DIR = path.resolve(ROOT_DIR, "content-crag");
+const DOCS_DIR = path.resolve(ROOT_DIR, "content");
 interface LinkElement extends Node {
   type: "link" | "image";
   url: string;

docs/next/package.json

@@ -3,13 +3,13 @@
   "version": "0.1.0",
   "private": true,
   "scripts": {
-    "dev": "PORT=3001 next-remote-watch ../content-crag",
+    "dev": "PORT=3001 next-remote-watch ../content",
     "build": "next build",
     "build-master": "VERSIONING_DISABLED=true next build",
     "start": "next start",
     "snapshot": "ts-node -O '{\"module\": \"commonjs\"}' scripts/snapshot.ts && yarn format",
     "test": "jest",
-    "format": "prettier --config ./.prettierrc.js --write \"../content-crag/**/*.mdx\" --loglevel warn && remark --rc-path ./.remarkrc.js \"../content-crag/**/*.mdx\" --output --silent"
+    "format": "prettier --config ./.prettierrc.js --write \"../content/**/*.mdx\" --loglevel warn && remark --rc-path ./.remarkrc.js \"../content/**/*.mdx\" --output --silent"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.18.0",

docs/next/pages/[...page].tsx

@@ -289,7 +289,7 @@ async function getVersionedContent(version: string, asPath: string) {
 async function getContent(version: string, asPath: string) {
   if (version == "master") {
     // render files from the local content folder
-    const basePath = path.resolve("../content-crag");
+    const basePath = path.resolve("../content");
     const pathToFile = path.join(basePath, asPath);
     const buffer = await fs.readFile(pathToFile);
     const contentString = buffer.toString();

docs/next/scripts/snapshot.ts

@@ -13,7 +13,7 @@ import { parse as yaml } from "yaml";
 
 // Main
 (async () => {
-  const stream = fg.stream(["../content-crag/**/*.mdx"]);
+  const stream = fg.stream(["../content/**/*.mdx"]);
 
   let stats: SnapshotStats & ImageStats = {
     totalSnapshots: 0,

docs/next/tailwind.config.js

@@ -7,7 +7,7 @@ module.exports = {
     "./pages/**/*.{js,jsx,ts,tsx}",
     "./components/**/*.{js,jsx,ts,tsx}",
     "./layouts/**/*.{js,jsx,ts,tsx}",
-    "../content-crag/**/*.mdx",
+    "../content/**/*.mdx",
   ],
   darkMode: "class",
   variants: {

docs/next/util/codeTransformer.ts

@@ -9,7 +9,7 @@ const DAGSTER_REPO =
 
 const DOCS_SNIPPET = path.join(
   DAGSTER_REPO,
-  "/examples/docs_snippets_crag/docs_snippets_crag"
+  "/examples/docs_snippets/docs_snippets"
 );
 
 export interface SnapshotStats {

docs/next/util/useNavigation.ts

@@ -1,4 +1,4 @@
-import masterNavigation from "../../content-crag/_navigation.json";
+import masterNavigation from "../../content/_navigation.json";
 import { useVersion } from "./useVersion";
 import versionedNavigation from "../.versioned_content/_versioned_navigation.json";
 

docs/pack_json.py

@@ -94,7 +94,7 @@ def copy_searchindex(src_dir, dest_dir, src_file="searchindex.json", dest_file="
 
 def main():
     json_directory = file_relative_path(__file__, "sphinx/_build/json")
-    content_dir = file_relative_path(__file__, "./content-crag/api")
+    content_dir = file_relative_path(__file__, "./content/api")
 
     directories_to_pack = {
         os.path.join(json_directory, "sections"): "sections.json",

docs/update_version.py

@@ -8,7 +8,7 @@
 import click
 from dagster import file_relative_path
 
-CONTENT_DIR = file_relative_path(__file__, "./content-crag")
+CONTENT_DIR = file_relative_path(__file__, "./content")
 IMAGE_DIR = file_relative_path(__file__, "next/public/images")
 VERSIONED_DIR = file_relative_path(__file__, "next/.versioned_content")
 

examples/docs_snippets_crag/docs_snippets_crag/concepts/configuration/execute_with_config.py → examples/docs_snippets/docs_snippets/concepts/configuration/execute_with_config.py

@@ -1,5 +1,5 @@
-from docs_snippets_crag.concepts.configuration.configurable_op import config_example
-from docs_snippets_crag.concepts.configuration.configurable_op_with_schema import nests_configurable
+from docs_snippets.concepts.configuration.configurable_op import config_example
+from docs_snippets.concepts.configuration.configurable_op_with_schema import nests_configurable
 
 
 def execute_with_config():

examples/docs_snippets_crag/docs_snippets_crag/concepts/partitions_schedules_sensors/partitioned_config_test.py → examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/partitioned_config_test.py

@@ -1,7 +1,7 @@
 """isort:skip_file"""
 
 
-from docs_snippets_crag.concepts.partitions_schedules_sensors.partitioned_job import (
+from docs_snippets.concepts.partitions_schedules_sensors.partitioned_job import (
     do_stuff_partitioned,
     my_partitioned_config,
 )

examples/docs_snippets_crag/docs_snippets_crag/concepts/partitions_schedules_sensors/partitioned_job_test.py → examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/partitioned_job_test.py

@@ -1,6 +1,6 @@
 """isort:skip_file"""
 
-from docs_snippets_crag.concepts.partitions_schedules_sensors.partitioned_job import (
+from docs_snippets.concepts.partitions_schedules_sensors.partitioned_job import (
     do_stuff_partitioned,
 )
 

examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/workspace.yaml

@@ -0,0 +1,2 @@
+load_from:
+  - python_module: docs_snippets.concepts.partitions_schedules_sensors.repo

examples/docs_snippets_crag/docs_snippets_crag/deploying/airflow/mounted.py → examples/docs_snippets/docs_snippets/deploying/airflow/mounted.py

@@ -1,7 +1,7 @@
 from dagster_airflow.factory import make_airflow_dag
 
 dag, steps = make_airflow_dag(
-    module_name="docs_snippets_crag.intro_tutorial.airflow",
+    module_name="docs_snippets.intro_tutorial.airflow",
     pipeline_name="hello_cereal_pipeline",
     run_config={"storage": {"filesystem": {"config": {"base_dir": "/container_tmp"}}}},
     dag_id=None,

examples/docs_snippets_crag/docs_snippets_crag/guides/dagster/reexecution/reexecution_api.py → examples/docs_snippets/docs_snippets/guides/dagster/reexecution/reexecution_api.py

@@ -1,5 +1,5 @@
 from dagster import DagsterInstance, execute_pipeline, reexecute_pipeline
-from docs_snippets_crag.guides.dagster.reexecution.unreliable_job import unreliable_job
+from docs_snippets.guides.dagster.reexecution.unreliable_job import unreliable_job
 
 instance = DagsterInstance.ephemeral()