Add support for upgrading version 3 configurations to version 4 (#452)

### What

This PR adds the ability to upgrade a configuration directory in version
3 to an equivalent one in version 4.

The cli now supports the command `cli upgrade --from-dir <existing
configuration dir> --to-dir <new configuration dir>`, with the intended
semantics that it upgrades the configuration in `<existing configuration
dir>` to the newest supported version.

### How

Many places in the code (though mostly tests) is tightly coupled with
internal details of version 3 and will attempt to construct and
manipulate configurations directly.

This PR attempts to define a version-agnostic interface that can be used
instead.

Cargo.toml

@@ -2,7 +2,9 @@
 resolver = "2"
 
 package.version = "0.6.0"
+
 package.edition = "2021"
+
 package.license = "Apache-2.0"
 
 members = [
@@ -29,6 +31,5 @@ similar_names = "allow"
 too_many_lines = "allow"
 
 [workspace.dependencies]
-ndc-models = { git = "https://github.com/hasura/ndc-spec.git", tag = "v0.1.2" }
 ndc-sdk = { git = "https://github.com/hasura/ndc-sdk-rs.git", rev = "6158b1adbcc4ac7d8acb721c1626f6f715424a27" }
 ndc-test = { git = "https://github.com/hasura/ndc-spec.git", tag = "v0.1.2" }

crates/cli/Cargo.toml

@@ -12,9 +12,7 @@ ndc-postgres-configuration = { path = "../configuration" }
 
 anyhow = "1.0.82"
 clap = { version = "4.5.4", features = ["derive", "env"] }
-schemars = { version = "0.8.17", features = ["smol_str", "preserve_order"] }
 serde = { version = "1.0.200", features = ["derive"] }
-serde_json = { version = "1.0.116", features = ["raw_value"] }
 serde_yaml = "0.9.34"
 thiserror = "1.0.59"
 tokio = { version = "1.37.0", features = ["full"] }

crates/cli/src/lib.rs

@@ -33,6 +33,12 @@ pub enum Command {
     },
     /// Update the configuration by introspecting the database, using the configuration options.
     Update,
+    /// Upgrade the configuration to the latest version. This does not involve the database.
+    Upgrade {
+        #[arg(long)]
+        dir_from: PathBuf,
+        dir_to: PathBuf,
+    },
 }
 
 /// The set of errors that can go wrong _in addition to_ generic I/O or parsing errors.
@@ -47,6 +53,7 @@ pub async fn run(command: Command, context: Context<impl Environment>) -> anyhow
     match command {
         Command::Initialize { with_metadata } => initialize(with_metadata, context).await?,
         Command::Update => update(context).await?,
+        Command::Upgrade { dir_from, dir_to } => upgrade(dir_from, dir_to).await?,
     };
     Ok(())
 }
@@ -60,33 +67,15 @@ pub async fn run(command: Command, context: Context<impl Environment>) -> anyhow
 /// Optionally, this can also create the connector metadata, which is used by the Hasura CLI to
 /// automatically work with this CLI as a plugin.
 async fn initialize(with_metadata: bool, context: Context<impl Environment>) -> anyhow::Result<()> {
-    let configuration_file = context
-        .context_path
-        .join(configuration::CONFIGURATION_FILENAME);
-    fs::create_dir_all(&context.context_path).await?;
-
     // refuse to initialize the directory unless it is empty
     let mut items_in_dir = fs::read_dir(&context.context_path).await?;
     if items_in_dir.next_entry().await?.is_some() {
         Err(Error::DirectoryIsNotEmpty)?;
     }
 
-    // create the configuration file
-    fs::write(
-        configuration_file,
-        serde_json::to_string_pretty(&configuration::RawConfiguration::empty())? + "\n",
-    )
-    .await?;
-
-    // create the jsonschema file
-    let configuration_jsonschema_file_path = context
-        .context_path
-        .join(configuration::CONFIGURATION_JSONSCHEMA_FILENAME);
-
-    let output = schemars::schema_for!(ndc_postgres_configuration::RawConfiguration);
-    fs::write(
-        &configuration_jsonschema_file_path,
-        serde_json::to_string_pretty(&output)? + "\n",
+    configuration::write_parsed_configuration(
+        configuration::ParsedConfiguration::initial(),
+        &context.context_path,
     )
     .await?;
 
@@ -139,35 +128,23 @@ async fn update(context: Context<impl Environment>) -> anyhow::Result<()> {
     // We want to detect this scenario and retry, or fail if we are unable to.
     // We do that with a few attempts.
     for _attempt in 1..=UPDATE_ATTEMPTS {
-        let configuration_file_path = context
-            .context_path
-            .join(configuration::CONFIGURATION_FILENAME);
-        let input: configuration::RawConfiguration = {
-            let configuration_file_contents =
-                read_config_file_contents(&configuration_file_path).await?;
-            serde_json::from_str(&configuration_file_contents)?
-        };
-        let output = configuration::introspect(input.clone(), &context.environment).await?;
+        let existing_configuration =
+            configuration::parse_configuration(&context.context_path).await?;
+        let output =
+            configuration::introspect(existing_configuration.clone(), &context.environment).await?;
 
         // Check that the input file did not change since we started introspecting,
-        let input_again_before_write: configuration::RawConfiguration = {
-            let configuration_file_contents =
-                read_config_file_contents(&configuration_file_path).await?;
-            serde_json::from_str(&configuration_file_contents)?
-        };
+        let input_again_before_write =
+            configuration::parse_configuration(&context.context_path).await?;
 
         // and skip this attempt if it has.
-        if input_again_before_write == input {
+        if input_again_before_write == existing_configuration {
             // If the introspection result is different than the current config,
             // change it. Otherwise, continue.
-            if input == output {
+            if existing_configuration == output {
                 // The configuration is up-to-date. Nothing to do.
             } else {
-                fs::write(
-                    &configuration_file_path,
-                    serde_json::to_string_pretty(&output)? + "\n",
-                )
-                .await?;
+                configuration::write_parsed_configuration(output, &context.context_path).await?;
             }
             return Ok(());
         }
@@ -181,17 +158,12 @@ async fn update(context: Context<impl Environment>) -> anyhow::Result<()> {
     ))
 }
 
-async fn read_config_file_contents(configuration_file_path: &PathBuf) -> anyhow::Result<String> {
-    fs::read_to_string(configuration_file_path)
-        .await
-        .map_err(|err| {
-            if err.kind() == std::io::ErrorKind::NotFound {
-                anyhow::anyhow!(
-                    "{}: No such file or directory.",
-                    configuration_file_path.display()
-                )
-            } else {
-                err.into()
-            }
-        })
+/// Upgrade the configuration in a directory by trying to read it and then write it back
+/// out to a different directory.
+///
+async fn upgrade(dir_from: PathBuf, dir_to: PathBuf) -> anyhow::Result<()> {
+    let old_configuration = configuration::parse_configuration(dir_from).await?;
+    let upgraded_configuration = configuration::upgrade_to_latest_version(old_configuration);
+    configuration::write_parsed_configuration(upgraded_configuration, dir_to).await?;
+    Ok(())
 }

crates/cli/tests/initialize_tests.rs

@@ -4,7 +4,7 @@ use tokio::fs;
 
 use ndc_postgres_cli::*;
 use ndc_postgres_configuration as configuration;
-use ndc_postgres_configuration::RawConfiguration;
+use ndc_postgres_configuration::ParsedConfiguration;
 
 #[tokio::test]
 async fn test_initialize_directory() -> anyhow::Result<()> {
@@ -31,7 +31,7 @@ async fn test_initialize_directory() -> anyhow::Result<()> {
     assert!(configuration_file_path.exists());
     let contents = fs::read_to_string(configuration_file_path).await?;
     common::assert_ends_with_newline(&contents);
-    let _: RawConfiguration = serde_json::from_str(&contents)?;
+    let _: ParsedConfiguration = configuration::parse_configuration(&dir).await?;
 
     let metadata_file_path = dir
         .path()

crates/cli/tests/update_tests.rs

@@ -5,7 +5,7 @@ use tokio::fs;
 use ndc_postgres_cli::*;
 use ndc_postgres_configuration as configuration;
 use ndc_postgres_configuration::environment::FixedEnvironment;
-use ndc_postgres_configuration::RawConfiguration;
+use ndc_postgres_configuration::ParsedConfiguration;
 
 const CONNECTION_URI: &str = "postgresql://postgres:password@localhost:64002";
 
@@ -18,17 +18,16 @@ async fn test_update_configuration() -> anyhow::Result<()> {
     });
 
     {
-        let configuration_file_path = dir.path().join("configuration.json");
         let connection_settings =
             configuration::version3::connection_settings::DatabaseConnectionSettings {
                 connection_uri: connection_uri.clone(),
                 ..configuration::version3::connection_settings::DatabaseConnectionSettings::empty()
             };
-        let input = RawConfiguration::Version3(configuration::version3::RawConfiguration {
+        let input = ParsedConfiguration::Version3(configuration::version3::RawConfiguration {
             connection_settings,
             ..configuration::version3::RawConfiguration::empty()
         });
-        fs::write(configuration_file_path, serde_json::to_string(&input)?).await?;
+        configuration::write_parsed_configuration(input, &dir).await?;
     }
 
     let environment =
@@ -44,9 +43,9 @@ async fn test_update_configuration() -> anyhow::Result<()> {
     assert!(configuration_file_path.exists());
     let contents = fs::read_to_string(configuration_file_path).await?;
     common::assert_ends_with_newline(&contents);
-    let output: RawConfiguration = serde_json::from_str(&contents)?;
+    let output: ParsedConfiguration = configuration::parse_configuration(&dir).await?;
     match output {
-        RawConfiguration::Version3(configuration::version3::RawConfiguration {
+        ParsedConfiguration::Version3(configuration::version3::RawConfiguration {
             connection_settings,
             metadata,
             ..
@@ -55,7 +54,7 @@ async fn test_update_configuration() -> anyhow::Result<()> {
             let some_table_metadata = metadata.tables.0.get("Artist");
             assert!(some_table_metadata.is_some());
         }
-        RawConfiguration::Version4(_) => panic!("Expected version 3"),
+        ParsedConfiguration::Version4(_) => panic!("Expected version 3"),
     }
 
     Ok(())