Create unified storage configuration for dagster.yaml (#7283)

* Add legacy stubs, unified storage configuration

* lint

* lint

* fix legacy

* add scheme to pg config

docs/content/deployment/guides/aws.mdx

@@ -16,38 +16,14 @@ To host Dagster on a bare VM or in Docker on EC2, see [Running Dagster as a serv
 You can use a hosted RDS PostgreSQL database for your Dagster run/events data by configuring your `dagster.yaml` file.
 
 ```python file=/deploying/dagster-pg.yaml
-run_storage:
-  module: dagster_postgres.run_storage
-  class: PostgresRunStorage
-  config:
-    postgres_db:
-      username: { username }
-      password: { password }
-      hostname: { hostname }
-      db_name: { database }
-      port: { port }
-
-event_log_storage:
-  module: dagster_postgres.event_log
-  class: PostgresEventLogStorage
-  config:
-    postgres_db:
-      username: { username }
-      password: { password }
-      hostname: { hostname }
-      db_name: { db_name }
-      port: { port }
-
-schedule_storage:
-  module: dagster_postgres.schedule_storage
-  class: PostgresScheduleStorage
-  config:
+storage:
+  postgres:
     postgres_db:
-      username: { username }
-      password: { password }
-      hostname: { hostname }
-      db_name: { db_name }
-      port: { port }
+      username: my_username
+      password: my_password
+      hostname: my_hostname
+      db_name: my_database
+      port: 5432
 ```
 
 In this case, you'll want to ensure you provide the right connection strings for your RDS instance, and ensure that the node or container hosting Dagit is able to connect to RDS.

docs/content/deployment/guides/celery.mdx

@@ -72,38 +72,14 @@ In production, more configuration is required.
 First, ensure that appropriate persistent run and event log storage, e.g., **`PostgresRunStorage`** and **`PostgresEventLogStorage`**, are configured on your Dagster instance, so that Dagit and the workers can communicate information about the run and events. You can do this by adding a block such as the following to your `dagster.yaml` (by default, Dagster will look for this file at `$DAGSTER_HOME/dagster.yaml` when the `DAGSTER_HOME` environment variable is set):
 
 ```python file=/deploying/dagster-pg.yaml
-run_storage:
-  module: dagster_postgres.run_storage
-  class: PostgresRunStorage
-  config:
+storage:
+  postgres:
     postgres_db:
-      username: { username }
-      password: { password }
-      hostname: { hostname }
-      db_name: { database }
-      port: { port }
-
-event_log_storage:
-  module: dagster_postgres.event_log
-  class: PostgresEventLogStorage
-  config:
-    postgres_db:
-      username: { username }
-      password: { password }
-      hostname: { hostname }
-      db_name: { db_name }
-      port: { port }
-
-schedule_storage:
-  module: dagster_postgres.schedule_storage
-  class: PostgresScheduleStorage
-  config:
-    postgres_db:
-      username: { username }
-      password: { password }
-      hostname: { hostname }
-      db_name: { db_name }
-      port: { port }
+      username: my_username
+      password: my_password
+      hostname: my_hostname
+      db_name: my_database
+      port: 5432
 ```
 
 **The same instance config must be present in Dagit's environment and in the workers' environments.** If you haven't already, please read about [the Dagster instance](/deployment/dagster-instance).

docs/content/deployment/guides/docker.mdx

@@ -42,40 +42,8 @@ load_from:
 as well as a `dagster.yaml` file to configure your [Dagster instance](/deployment/dagster-instance):
 
 ```yaml file=/deploying/docker/dagster.yaml
-run_storage:
-  module: dagster_postgres.run_storage
-  class: PostgresRunStorage
-  config:
-    postgres_db:
-      username:
-        env: DAGSTER_PG_USERNAME
-      password:
-        env: DAGSTER_PG_PASSWORD
-      hostname:
-        env: DAGSTER_PG_HOST
-      db_name:
-        env: DAGSTER_PG_DB
-      port: 5432
-
-event_log_storage:
-  module: dagster_postgres.event_log
-  class: PostgresEventLogStorage
-  config:
-    postgres_db:
-      username:
-        env: DAGSTER_PG_USERNAME
-      password:
-        env: DAGSTER_PG_PASSWORD
-      hostname:
-        env: DAGSTER_PG_HOST
-      db_name:
-        env: DAGSTER_PG_DB
-      port: 5432
-
-schedule_storage:
-  module: dagster_postgres.schedule_storage
-  class: PostgresScheduleStorage
-  config:
+storage:
+  postgres:
     postgres_db:
       username:
         env: DAGSTER_PG_USERNAME

docs/content/deployment/guides/gcp.mdx

@@ -16,38 +16,14 @@ To host Dagit or Dagster Daemon on a bare VM or in Docker on GCE, see [Running D
 We recommend launching a Cloud SQL PostgreSQL instance for run and events data. You can configure Dagit to use Cloud SQL to run and events data by setting blocks in your `$DAGSTER_HOME/dagster.yaml` appropriately:
 
 ```python file=/deploying/dagster-pg.yaml
-run_storage:
-  module: dagster_postgres.run_storage
-  class: PostgresRunStorage
-  config:
+storage:
+  postgres:
     postgres_db:
-      username: { username }
-      password: { password }
-      hostname: { hostname }
-      db_name: { database }
-      port: { port }
-
-event_log_storage:
-  module: dagster_postgres.event_log
-  class: PostgresEventLogStorage
-  config:
-    postgres_db:
-      username: { username }
-      password: { password }
-      hostname: { hostname }
-      db_name: { db_name }
-      port: { port }
-
-schedule_storage:
-  module: dagster_postgres.schedule_storage
-  class: PostgresScheduleStorage
-  config:
-    postgres_db:
-      username: { username }
-      password: { password }
-      hostname: { hostname }
-      db_name: { db_name }
-      port: { port }
+      username: my_username
+      password: my_password
+      hostname: my_hostname
+      db_name: my_database
+      port: 5432
 ```
 
 In this case, you'll want to ensure you provide the right connection strings for your Cloud SQL instance, and that the node or container hosting Dagit is able to connect to Cloud SQL.

docs/sphinx/sections/api/apidocs/internals.rst

@@ -71,6 +71,15 @@ Instance
 
 ----
 
+Storage
+-------
+
+.. currentmodule:: dagster.core.storage.base_storage
+
+.. autoclass:: DagsterStorage
+
+----
+
 Run storage
 -----------
 

examples/docs_snippets/docs_snippets/deploying/dagster-mysql-legacy.yaml

@@ -0,0 +1,44 @@
+# start_marker_runs
+
+run_storage:
+ module: dagster_mysql.run_storage
+ class: MySQLRunStorage
+ config:
+  mysql_db:
+   username: { username }
+   password: { password }
+   hostname: { hostname }
+   db_name: { database }
+   port: { port }
+
+# end_marker_runs
+
+# start_marker_event_log
+
+event_log_storage:
+ module: dagster_mysql.event_log
+ class: MySQLEventLogStorage
+ config:
+  mysql_db:
+   username: { username }
+   password: { password }
+   hostname: { hostname }
+   db_name: { db_name }
+   port: { port }
+
+# end_marker_event_log
+
+# start_marker_schedules
+
+schedule_storage:
+ module: dagster_mysql.schedule_storage
+ class: MySQLScheduleStorage
+ config:
+  mysql_db:
+   username: { username }
+   password: { password }
+   hostname: { hostname }
+   db_name: { db_name }
+   port: { port }
+
+# end_marker_schedules

examples/docs_snippets/docs_snippets/deploying/dagster-mysql.yaml

@@ -1,44 +1,8 @@
-# start_marker_runs
-
-run_storage:
- module: dagster_mysql.run_storage
- class: MySQLRunStorage
- config:
-  mysql_db:
-   username: { username }
-   password: { password }
-   hostname: { hostname }
-   db_name: { database }
-   port: { port }
-
-# end_marker_runs
-
-# start_marker_event_log
-
-event_log_storage:
- module: dagster_mysql.event_log
- class: MySQLEventLogStorage
- config:
-  mysql_db:
-   username: { username }
-   password: { password }
-   hostname: { hostname }
-   db_name: { db_name }
-   port: { port }
-
-# end_marker_event_log
-
-# start_marker_schedules
-
-schedule_storage:
- module: dagster_mysql.schedule_storage
- class: MySQLScheduleStorage
- config:
-  mysql_db:
-   username: { username }
-   password: { password }
-   hostname: { hostname }
-   db_name: { db_name }
-   port: { port }
-
-# end_marker_schedules
+storage:
+  mysql:
+    mysql_db:
+    username: { username }
+    password: { password }
+    hostname: { hostname }
+    db_name: { database }
+    port: { port }

examples/docs_snippets/docs_snippets/deploying/dagster-pg-legacy.yaml

@@ -0,0 +1,32 @@
+run_storage:
+  module: dagster_postgres.run_storage
+  class: PostgresRunStorage
+  config:
+    postgres_db:
+      username: { username }
+      password: { password }
+      hostname: { hostname }
+      db_name: { db_name }
+      port: { port }
+
+event_log_storage:
+  module: dagster_postgres.event_log
+  class: PostgresEventLogStorage
+  config:
+    postgres_db:
+      username: { username }
+      password: { password }
+      hostname: { hostname }
+      db_name: { db_name }
+      port: { port }
+
+schedule_storage:
+  module: dagster_postgres.event_log
+  class: PostgresScheduleStorage
+  config:
+    postgres_db:
+      username: { username }
+      password: { password }
+      hostname: { hostname }
+      db_name: { db_name }
+      port: { port }

examples/docs_snippets/docs_snippets/deploying/dagster-pg.yaml

@@ -1,32 +1,8 @@
-run_storage:
-  module: dagster_postgres.run_storage
-  class: PostgresRunStorage
-  config:
+storage:
+  postgres:
     postgres_db:
-      username: { username }
-      password: { password }
-      hostname: { hostname }
-      db_name: { database }
-      port: { port }
-
-event_log_storage:
-  module: dagster_postgres.event_log
-  class: PostgresEventLogStorage
-  config:
-    postgres_db:
-      username: { username }
-      password: { password }
-      hostname: { hostname }
-      db_name: { db_name }
-      port: { port }
-
-schedule_storage:
-  module: dagster_postgres.schedule_storage
-  class: PostgresScheduleStorage
-  config:
-    postgres_db:
-      username: { username }
-      password: { password }
-      hostname: { hostname }
-      db_name: { db_name }
-      port: { port }
+      username: my_username
+      password: my_password
+      hostname: my_hostname
+      db_name: my_database
+      port: 5432