interpolate vars

crap

maybe now

ok

bin/cloudflare_worker/src/lib.rs

@@ -3,65 +3,84 @@ use std::str::FromStr;
 use conductor_common::http::{
     ConductorHttpRequest, HeaderName, HeaderValue, HttpHeadersMap, Method,
 };
-use conductor_config::parse_config_from_yaml;
+use conductor_config::interpolate::ConductorEnvVars;
+use conductor_config::parse_config_contents;
 use conductor_engine::gateway::ConductorGateway;
 use std::panic;
 use tracing_subscriber::fmt::time::UtcTime;
 use tracing_subscriber::prelude::*;
 use tracing_web::MakeConsoleWriter;
 use worker::*;
 
+struct EnvVarsFetcher<'a> {
+    env: &'a Env,
+}
+
+impl<'a> EnvVarsFetcher<'a> {
+    pub fn new(env: &'a Env) -> Self {
+        Self { env }
+    }
+}
+
+impl<'a> ConductorEnvVars for EnvVarsFetcher<'a> {
+    fn get_var(&self, key: &str) -> Option<String> {
+        self.env.var(key).map(|s| s.to_string()).ok()
+    }
+}
+
 async fn run_flow(mut req: Request, env: Env, _ctx: Context) -> Result<Response> {
     let conductor_config_str = env.var("CONDUCTOR_CONFIG").map(|v| v.to_string());
-
+    let env_fetcher: EnvVarsFetcher<'_> = EnvVarsFetcher::new(&env);
     match conductor_config_str {
-        Ok(conductor_config_str) => match parse_config_from_yaml(&conductor_config_str) {
-            Ok(conductor_config) => {
-                let gw = ConductorGateway::lazy(conductor_config);
-
-                if let Some(route_data) = gw.match_route(&req.url().unwrap()) {
-                    console_log!("Route found: {:?}", route_data);
-                    let mut headers_map = HttpHeadersMap::new();
-
-                    for (k, v) in req.headers().entries() {
-                        headers_map.insert(
-                            HeaderName::from_str(&k).unwrap(),
-                            HeaderValue::from_str(&v).unwrap(),
-                        );
-                    }
-
-                    let body = req.bytes().await.unwrap().into();
-                    let uri = req.url().unwrap().to_string();
-                    let query_string = req.url().unwrap().query().unwrap_or_default().to_string();
-                    let method = Method::from_str(req.method().as_ref()).unwrap();
-
-                    let conductor_req = ConductorHttpRequest {
-                        body,
-                        uri,
-                        query_string,
-                        method,
-                        headers: headers_map,
-                    };
-
-                    let conductor_response = gw.execute(conductor_req, &route_data).await;
-
-                    let mut response_headers = Headers::new();
-                    for (k, v) in conductor_response.headers.into_iter() {
-                        response_headers
-                            .append(k.unwrap().as_str(), v.to_str().unwrap())
-                            .unwrap();
-                    }
-
-                    Response::from_bytes(conductor_response.body.into()).map(|r| {
-                        r.with_status(conductor_response.status.as_u16())
-                            .with_headers(response_headers)
-                    })
-                } else {
-                    Response::error("No route found", 404)
+        Ok(conductor_config_str) => {
+            let conductor_config = parse_config_contents(
+                conductor_config_str,
+                conductor_config::ConfigFormat::Yaml,
+                env_fetcher,
+            );
+
+            let gw = ConductorGateway::lazy(conductor_config);
+
+            if let Some(route_data) = gw.match_route(&req.url().unwrap()) {
+                let mut headers_map = HttpHeadersMap::new();
+
+                for (k, v) in req.headers().entries() {
+                    headers_map.insert(
+                        HeaderName::from_str(&k).unwrap(),
+                        HeaderValue::from_str(&v).unwrap(),
+                    );
                 }
+
+                let body = req.bytes().await.unwrap().into();
+                let uri = req.url().unwrap().to_string();
+                let query_string = req.url().unwrap().query().unwrap_or_default().to_string();
+                let method = Method::from_str(req.method().as_ref()).unwrap();
+
+                let conductor_req = ConductorHttpRequest {
+                    body,
+                    uri,
+                    query_string,
+                    method,
+                    headers: headers_map,
+                };
+
+                let conductor_response = gw.execute(conductor_req, &route_data).await;
+
+                let mut response_headers = Headers::new();
+                for (k, v) in conductor_response.headers.into_iter() {
+                    response_headers
+                        .append(k.unwrap().as_str(), v.to_str().unwrap())
+                        .unwrap();
+                }
+
+                Response::from_bytes(conductor_response.body.into()).map(|r| {
+                    r.with_status(conductor_response.status.as_u16())
+                        .with_headers(response_headers)
+                })
+            } else {
+                Response::error("No route found", 404)
             }
-            Err(e) => Response::error(e.to_string(), 500),
-        },
+        }
         Err(e) => Response::error(e.to_string(), 500),
     }
 }

bin/conductor/src/lib.rs

@@ -1,3 +1,5 @@
+use std::{collections::HashMap, env::vars};
+
 use actix_web::{
     body::MessageBody,
     dev::{Response, ServiceFactory, ServiceRequest, ServiceResponse},
@@ -6,16 +8,34 @@ use actix_web::{
     App, Error, HttpRequest, HttpResponse, HttpServer, Responder, Scope,
 };
 use conductor_common::http::{ConductorHttpRequest, HttpHeadersMap};
-use conductor_config::{load_config, ConductorConfig};
+use conductor_config::{interpolate::ConductorEnvVars, load_config, ConductorConfig};
 use conductor_engine::gateway::{ConductorGateway, ConductorGatewayRouteData};
 use tracing::debug;
 use tracing_subscriber::fmt::format::FmtSpan;
 
+struct EnvVarsFetcher {
+    vars_map: HashMap<String, String>,
+}
+
+impl EnvVarsFetcher {
+    pub fn new() -> Self {
+        Self {
+            vars_map: vars().collect::<HashMap<String, String>>(),
+        }
+    }
+}
+
+impl ConductorEnvVars for EnvVarsFetcher {
+    fn get_var(&self, key: &str) -> Option<String> {
+        self.vars_map.get(key).cloned()
+    }
+}
+
 pub async fn run_services(config_file_path: &String) -> std::io::Result<()> {
     println!("gateway process started");
     println!("loading configuration from {}", config_file_path);
-    let config_object = load_config(config_file_path).await;
-    println!("configuration loaded");
+    let config_object = load_config(config_file_path, EnvVarsFetcher::new()).await;
+    println!("configuration loaded and parsed");
 
     let logger_config = config_object.logger.clone();
     tracing_subscriber::fmt()

libs/config/Cargo.toml

@@ -16,3 +16,5 @@ tracing = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 serde_yaml = "0.9.27"
+regex = "1.10.2"
+lazy_static = "1.4.0"

libs/config/conductor.schema.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json-schema.org/draft/2019-09/schema",
   "title": "ConductorConfig",
-  "description": "This section describes the top-level configuration object for Conductor gateway.\n\nConductor supports both YAML and JSON format for the configuration file.\n\n## Loading the config file\n\nThe configuration is loaded when server starts, based on the runtime environment you are using:\n\n### Binary\n\nIf you are running the Conductor binary directly, you can specify the configuration file path using the first argument:\n\n```\n\n./conductor my-config-file.json\n\n```\n\n> By default, Conductor will look for a file named `config.json` in the current directory.\n\n### Docker\n\nIf you are using Docker environment, you can mount the configuration file into the container, and then point the Conductor binary to it:\n\n```\n\ndocker run -v my-config-file.json:/app/config.json the-guild-org/conductor-t2:latest /app/config.json\n\n```\n\n### CloudFlare Worker\n\nWASM runtime doesn't allow filesystem access, so you need to load the configuration file into an environment variable named `CONDUCTOR_CONFIG`.\n\n## Autocomplete/validation in VSCode\n\nFor JSON files, you can specify the `$schema` property to enable autocomplete and validation in VSCode:\n\n```json filename=\"config.json\"\n\n{ \"$schema\": \"https://raw.githubusercontent.com/the-guild-org/conductor-t2/master/libs/config/conductor.schema.json\" }\n\n```\n\nFor YAML auto-complete and validation, you can install the [YAML Language Support](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml) extension and enable it by adding the following to your YAML file:\n\n```yaml filename=\"config.yaml\"\n\n$schema: \"https://raw.githubusercontent.com/the-guild-org/conductor-t2/master/libs/config/conductor.schema.json\"\n\n```\n\n### JSONSchema\n\nAs part of the release flow of Conductor, we are publishing the configuration schema as a JSONSchema file.\n\nYou can find [here the latest version of the schema](https://github.com/the-guild-org/conductor-t2/releases).",
+  "description": "This section describes the top-level configuration object for Conductor gateway.\n\nConductor supports both YAML and JSON format for the configuration file.\n\n## Loading the config file\n\nThe configuration is loaded when server starts, based on the runtime environment you are using:\n\n### Binary\n\nIf you are running the Conductor binary directly, you can specify the configuration file path using the first argument:\n\n```bash\n\nconductor my-config-file.json\n\n```\n\n> By default, Conductor will look for a file named `config.json` in the current directory.\n\n### Docker\n\nIf you are using Docker environment, you can mount the configuration file into the container, and then point the Conductor binary to it:\n\n```bash\n\ndocker run -v my-config-file.json:/app/config.json the-guild-org/conductor-t2:latest /app/config.json\n\n```\n\n### CloudFlare Worker\n\nWASM runtime doesn't allow filesystem access, so you need to load the configuration file into an environment variable named `CONDUCTOR_CONFIG`.\n\n## Autocomplete/validation in VSCode\n\nFor JSON files, you can specify the `$schema` property to enable autocomplete and validation in VSCode:\n\n```json filename=\"config.json\"\n\n{ \"$schema\": \"https://raw.githubusercontent.com/the-guild-org/conductor-t2/master/libs/config/conductor.schema.json\" }\n\n```\n\nFor YAML auto-complete and validation, you can install the [YAML Language Support](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml) extension and enable it by adding the following to your YAML file:\n\n```yaml filename=\"config.yaml\"\n\n$schema: \"https://raw.githubusercontent.com/the-guild-org/conductor-t2/master/libs/config/conductor.schema.json\"\n\n```\n\n### JSONSchema\n\nAs part of the release flow of Conductor, we are publishing the configuration schema as a JSONSchema file.\n\nYou can find [here the latest version of the schema](https://github.com/the-guild-org/conductor-t2/releases).\n\n### Environment Variables\n\nConductor supports environment variables interpolation in the configuration file.\n\nYou can use the `${VAR_NAME}` syntax to interpolate environment variables into the configuration file. A warning will be printed if the variable is not found.\n\nTo set a default value for an environment variable, you can use the `${VAR_NAME:default_value}` syntax.",
   "type": "object",
   "required": [
     "endpoints",

libs/config/src/interpolate.rs

@@ -0,0 +1,84 @@
+use lazy_static::lazy_static;
+use regex::{Captures, Regex};
+
+// The following file was shamefully copied from the Vector project:
+// https://github.com/vectordotdev/vector/blob/master/src/config/vars.rs
+// Interpolation is based on:
+// https://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap08.html
+
+lazy_static! {
+    pub static ref EMPTY_STRING: String = "".to_string();
+    pub static ref ENVIRONMENT_VARIABLE_INTERPOLATION_REGEX: Regex = Regex::new(
+        r"(?x)
+        \$\$|
+        \$([[:word:].]+)|
+        \$\{([[:word:].]+)(?:(:?-|:?\?)([^}]*))?\}",
+    )
+    .unwrap();
+}
+
+type Warnings = Vec<String>;
+type Errors = Vec<String>;
+
+pub trait ConductorEnvVars {
+    fn get_var(&self, key: &str) -> Option<String>;
+}
+
+pub fn interpolate(
+    input: &str,
+    env_fetcher: impl ConductorEnvVars,
+) -> Result<(String, Warnings), Errors> {
+    let mut errors = Vec::new();
+    let mut warnings = Vec::new();
+
+    let interpolated = ENVIRONMENT_VARIABLE_INTERPOLATION_REGEX
+        .replace_all(input, |caps: &Captures| {
+            let flags = caps.get(3).map(|m| m.as_str()).unwrap_or_default();
+            let def_or_err = caps.get(4).map(|m| m.as_str()).unwrap_or_default().to_string();
+
+            caps.get(1)
+                .or_else(|| caps.get(2))
+                .map(|m| m.as_str())
+                .map(|name| {
+                    let val = env_fetcher.get_var(name);
+                    match flags {
+                        ":-" => match val {
+                            Some(v) if !v.is_empty() => v,
+                            _ => def_or_err,
+                        },
+                        "-" => val.unwrap_or(def_or_err),
+                        ":?" => match val {
+                            Some(v) if !v.is_empty() => v,
+                            _ => {
+                                errors.push(format!(
+                                    "Non-empty env var required in config. name = {:?}, error = {:?}",
+                                    name, def_or_err
+                                ));
+                                EMPTY_STRING.to_owned()
+                            },
+                        }
+                        "?" => val.unwrap_or_else(|| {
+                            errors.push(format!(
+                                "Missing env var required in config. name = {:?}, error = {:?}",
+                                name, def_or_err
+                            ));
+                            EMPTY_STRING.to_owned()
+                        }),
+                        _ => val.unwrap_or_else(|| {
+                            warnings
+                                .push(format!("Unknown env var in config. name = {:?}", name));
+                            EMPTY_STRING.to_owned()
+                        }),
+                    }
+                })
+                .unwrap_or("$".to_string())
+                .to_string()
+        })
+        .into_owned();
+
+    if errors.is_empty() {
+        Ok((interpolated, warnings))
+    } else {
+        Err(errors)
+    }
+}