Read Native Query SQL from files (#372)

<!-- The PR description should answer 2 (maybe 3) important questions:
-->

### What

Currently we can specify native queries by inlining their SQL as a
string in the `sql` field.
We'd like to be able to reference a `.sql` file instead, so we can avoid
escaping characters and can use newlines.

The new config format will accept the following structures:

```json
"sql": "<inline sql>" (as before, but also:)

"sql": { "inline": "<inline sql>" }

"sql": { "file": "<relative file path>" }
```

### How

- We parse the `NativeQuerySql` which represents the SQL field using an
intermediate new type `NativeQuerySqlExternal`.
- as part of the parse_configuration step and after serde
deserialization, we traverse the metadata and convert
`NativeQuerySqlExternal` to `NativeQuerySql` by reading from file
(performing IO) if needed, and parsing to `NativeQueryParts`. We use the
configuration directory that the user supplied to locate the relative
path.

We replace the existing `sql` field in nativequeryinfo with an Either
external or internal representation. This is not ideal because it breaks
the parse don't validate principle.
We would like to mitigate this in the future by separating the
`RawConfiguration` types and the (execution time) `Configuration` types
completely.
We will do that as part of a larger ticket of redesigning the
configuration format so it can be split across multiple files.

Author: Gil Mizrahi

Cargo.lock

@@ -2,6 +2,11 @@
 
 ## [Unreleased]
 
+### Added
+
+- Support for reading Native Query SQL from files.
+  ([#372](https://github.com/hasura/ndc-postgres/pull/372))
+
 ### Changed
 
 - Do not print information about when the cli decides not to write to a file.

changelog.md

@@ -60,14 +60,29 @@ pub async fn parse_configuration(
     environment: impl Environment,
 ) -> Result<Configuration, Error> {
     let configuration_file = configuration_dir.as_ref().join(CONFIGURATION_FILENAME);
-    let configuration_file_contents = fs::read_to_string(&configuration_file).await?;
-    let configuration: version3::RawConfiguration =
+
+    let configuration_file_contents =
+        fs::read_to_string(&configuration_file)
+            .await
+            .map_err(|err| {
+                Error::IoErrorButStringified(format!("{}: {}", &configuration_file.display(), err))
+            })?;
+    let mut configuration: version3::RawConfiguration =
         serde_json::from_str(&configuration_file_contents).map_err(|error| Error::ParseError {
             file_path: configuration_file.clone(),
             line: error.line(),
             column: error.column(),
             message: error.to_string(),
         })?;
+    // look for native query sql file references and read from disk.
+    for native_query_sql in configuration.metadata.native_queries.0.values_mut() {
+        native_query_sql.sql = metadata::NativeQuerySqlEither::NativeQuerySql(
+            native_query_sql
+                .sql
+                .from_external(configuration_dir.as_ref())
+                .map_err(Error::IoErrorButStringified)?,
+        );
+    }
     let connection_uri =
         match configuration.connection_settings.connection_uri {
             ConnectionUri(Secret::Plain(uri)) => Ok(uri),

crates/configuration/src/configuration.rs

@@ -21,6 +21,10 @@ pub enum Error {
         file_path: std::path::PathBuf,
         message: String,
     },
+
+    #[error("I/O error: {0}")]
+    IoErrorButStringified(String),
+
     #[error("I/O error: {0}")]
     IoError(#[from] std::io::Error),
 }

crates/configuration/src/error.rs

@@ -253,6 +253,9 @@ impl<Env: Environment + Send + Sync> ConnectorSetup for PostgresSetup<Env> {
                     ]))
                 }
                 configuration::Error::IoError(inner) => connector::ParseError::IoError(inner),
+                configuration::Error::IoErrorButStringified(inner) => {
+                    connector::ParseError::Other(inner.into())
+                }
             })
 
         // Note that we don't log validation errors, because they are part of the normal business

crates/connectors/ndc-postgres/src/connector.rs

@@ -10,3 +10,6 @@ workspace = true
 [dependencies]
 schemars = { version = "0.8.16", features = ["smol_str"] }
 serde = { version = "1.0.197", features = ["derive"] }
+
+[dev-dependencies]
+serde_json = "1.0.114"