feat: add convert config command

Cargo.toml

@@ -367,6 +367,7 @@ proptest = "1.2"
 quickcheck = "1.0.3"
 lookup = { package = "vector-lookup", path = "lib/vector-lookup", features = ["test"] }
 reqwest = { version = "0.11", features = ["json"] }
+rstest = {version = "0.18.2"}
 tempfile = "3.6.0"
 test-generator = "0.3.1"
 tokio = { version = "1.32.0", features = ["test-util"] }

src/cli.rs

@@ -9,7 +9,7 @@ use crate::service;
 use crate::tap;
 #[cfg(feature = "api-client")]
 use crate::top;
-use crate::{config, generate, get_version, graph, list, unit_test, validate};
+use crate::{config, convert_config, generate, get_version, graph, list, unit_test, validate};
 use crate::{generate_schema, signal};
 
 #[derive(Parser, Debug)]
@@ -34,6 +34,7 @@ impl Opts {
             Some(SubCommand::Validate(_))
             | Some(SubCommand::Graph(_))
             | Some(SubCommand::Generate(_))
+            | Some(SubCommand::ConvertConfig(_))
             | Some(SubCommand::List(_))
             | Some(SubCommand::Test(_)) => {
                 if self.root.verbose == 0 {
@@ -241,6 +242,9 @@ pub enum SubCommand {
     /// Validate the target config, then exit.
     Validate(validate::Opts),
 
+    /// Convert a config (or a directory of configs) from one format to another.
+    ConvertConfig(convert_config::Opts),
+
     /// Generate a Vector configuration containing a list of components.
     Generate(generate::Opts),
 
@@ -290,6 +294,7 @@ impl SubCommand {
     ) -> exitcode::ExitCode {
         match self {
             Self::Config(c) => config::cmd(c),
+            Self::ConvertConfig(opts) => convert_config::cmd(opts),
             Self::Generate(g) => generate::cmd(g),
             Self::GenerateSchema => generate_schema::cmd(),
             Self::Graph(g) => graph::cmd(g),

src/config/format.rs

@@ -2,6 +2,7 @@
 
 #![deny(missing_docs, missing_debug_implementations)]
 
+use std::fmt;
 use std::path::Path;
 use std::str::FromStr;
 
@@ -35,6 +36,16 @@ impl FromStr for Format {
     }
 }
 
+impl fmt::Display for Format {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Format::Toml => write!(f, "toml"),
+            Format::Json => write!(f, "json"),
+            Format::Yaml => write!(f, "yaml"),
+        }
+    }
+}
+
 impl Format {
     /// Obtain the format from the file path using extension as a hint.
     pub fn from_path<T: AsRef<Path>>(path: T) -> Result<Self, T> {

src/config/mod.rs

@@ -9,6 +9,7 @@ use std::{
 };
 
 use indexmap::IndexMap;
+use serde::Serialize;
 pub use vector_config::component::{GenerateConfig, SinkDescription, TransformDescription};
 use vector_config::configurable_component;
 pub use vector_core::config::{
@@ -100,7 +101,7 @@ impl ConfigPath {
     }
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Serialize)]
 pub struct Config {
     #[cfg(feature = "api")]
     pub api: api::Options,

src/convert_config.rs

@@ -0,0 +1,220 @@
+use crate::config::{format, ConfigBuilder, Format};
+use clap::Parser;
+use colored::*;
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::str::FromStr;
+
+#[derive(Parser, Debug)]
+#[command(rename_all = "kebab-case")]
+pub struct Opts {
+    /// The input path. It can be a single file or a directory. If this points to a directory,
+    /// all files with a "toml", "yaml" or "json" extension will be converted.
+    #[arg(short, long)]
+    pub(crate) input_path: PathBuf,
+
+    /// The output file or directory to be created. This command will fail if the output directory exists.
+    #[arg(short, long)]
+    pub(crate) output_path: PathBuf,
+
+    /// The target format to which existing config files will be converted to.
+    #[arg(long, default_value = "yaml")]
+    pub(crate) output_format: Format,
+}
+
+fn check_paths(opts: &Opts) -> Result<(), String> {
+    let in_metadata = fs::metadata(&opts.input_path).expect(&format!(
+        "Failed to get metadata for: {:?}",
+        &opts.input_path
+    ));
+
+    if opts.output_path.exists() {
+        return Err(format!(
+            "Output path {:?} already exists. Please provide a non-existing output path.",
+            opts.output_path
+        ));
+    }
+
+    if opts.output_path.extension().is_none() {
+        if in_metadata.is_file() {
+            return Err(format!(
+                "{:?} points to a file but {:?} points to a directory.",
+                opts.input_path, opts.output_path
+            ));
+        }
+    } else {
+        if in_metadata.is_dir() {
+            return Err(format!(
+                "{:?} points to a directory but {:?} points to a file.",
+                opts.input_path, opts.output_path
+            ));
+        }
+    }
+
+    Ok(())
+}
+
+pub(crate) fn cmd(opts: &Opts) -> exitcode::ExitCode {
+    if let Err(e) = check_paths(opts) {
+        #[allow(clippy::print_stderr)]
+        {
+            eprintln!("{}", e.red());
+        }
+        return exitcode::SOFTWARE;
+    }
+
+    return if opts.input_path.is_file() && opts.output_path.extension().is_some() {
+        if let Some(base_dir) = opts.output_path.file_name() {
+            if let Err(_) = fs::metadata(base_dir) {
+                fs::create_dir_all(base_dir).expect(&format!(
+                    "Failed to create output dir(s): {:?}",
+                    &opts.output_path
+                ));
+            }
+        }
+
+        match convert_config(&opts.input_path, &opts.output_path, opts.output_format) {
+            Ok(_) => exitcode::OK,
+            Err(errors) => {
+                #[allow(clippy::print_stderr)]
+                {
+                    errors.iter().for_each(|e| eprintln!("{}", e.red()));
+                }
+                exitcode::SOFTWARE
+            }
+        }
+    } else {
+        match walk_dir_and_convert(&opts.input_path, &opts.output_path, opts.output_format) {
+            Ok(()) => {
+                #[allow(clippy::print_stdout)]
+                {
+                    println!(
+                        "Finished conversion(s). Results are in {:?}",
+                        opts.output_path
+                    );
+                }
+                exitcode::OK
+            }
+            Err(errors) => {
+                #[allow(clippy::print_stderr)]
+                {
+                    errors.iter().for_each(|e| eprintln!("{}", e.red()));
+                }
+                exitcode::SOFTWARE
+            }
+        }
+    };
+}
+
+fn convert_config(
+    input_path: &Path,
+    output_path: &Path,
+    output_format: Format,
+) -> Result<(), Vec<String>> {
+    let input_format = Format::from_str(
+        input_path
+            .extension()
+            .expect(&format!("Failed to get extension for: {input_path:?}"))
+            .to_str()
+            .expect("Failed to convert OsStr to &str for: {input_path:?}"),
+    )
+    .expect(&format!(
+        "Failed to convert extension to Format for: {input_path:?}"
+    ));
+
+    if input_format == output_format {
+        return Ok(());
+    }
+
+    println!("Converting input config: {input_path:?}");
+    let file_contents = fs::read_to_string(&input_path).map_err(|e| vec![e.to_string()])?;
+    println!("{file_contents:?}");
+    let builder: ConfigBuilder = format::deserialize(&file_contents, input_format)?;
+    let config = builder.build()?;
+    let output_string =
+        format::serialize(&config, output_format).map_err(|e| vec![e.to_string()])?;
+    println!("{output_path:?} \n {output_string}");
+    fs::write(&output_path, output_string).map_err(|e| vec![e.to_string()])?;
+    println!("Wrote converted config to {output_path:?}.");
+    Ok(())
+}
+
+fn walk_dir_and_convert(
+    input_path: &Path,
+    output_dir: &Path,
+    output_format: Format,
+) -> Result<(), Vec<String>> {
+    let mut errors = Vec::new();
+
+    if input_path.is_dir() {
+        for entry in fs::read_dir(input_path).expect(&format!("Failed to read dir: {input_path:?}"))
+        {
+            let entry_path = entry
+                .expect(&format!("Failed to get entry for dir: {input_path:?}"))
+                .path();
+            let filename_path = entry_path
+                .file_name()
+                .expect(&format!("Failed to get base dir: {entry_path:?}"));
+            let new_output_dir = output_dir.join(filename_path);
+            fs::create_dir(&new_output_dir)
+                .expect("Failed to create output dir: {new_output_dir:?}");
+
+            if let Err(new_errors) =
+                walk_dir_and_convert(&entry_path, &new_output_dir, output_format)
+            {
+                errors.extend(new_errors);
+            }
+        }
+    } else {
+        let output_path = output_dir.join(
+            input_path
+                .with_extension(output_format.to_string().as_str())
+                .file_name()
+                .ok_or_else(|| {
+                    vec![format!(
+                        "Cannot create output path for input: {input_path:?}"
+                    )]
+                })?,
+        );
+
+        if let Err(new_errors) = convert_config(&input_path, &output_path, output_format) {
+            errors.extend(new_errors);
+        }
+    }
+
+    if errors.is_empty() {
+        Ok(())
+    } else {
+        Err(errors)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::config::Format;
+    use crate::convert_config::walk_dir_and_convert;
+    use rstest::rstest;
+    use std::env;
+    use std::path::PathBuf;
+    use tempfile::tempdir;
+
+    fn test_data_dir() -> PathBuf {
+        PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()).join("tests/data/cmd/config")
+    }
+
+    #[rstest]
+    #[case(Format::Toml)]
+    #[case(Format::Json)]
+    #[case(Format::Yaml)]
+    #[test]
+    fn convert_all_from_dir(#[case] output_format: Format) {
+        println!("\n\nconvert_all_from_dir {output_format:?}");
+        let input_path = test_data_dir();
+        let output_path = tempdir()
+            .expect("Unable to create tempdir for config")
+            .into_path();
+
+        let result = walk_dir_and_convert(&input_path, &output_path, output_format);
+        assert!(result.is_ok());
+    }
+}

src/lib.rs

@@ -67,6 +67,7 @@ pub mod aws;
 #[allow(unreachable_pub)]
 pub mod codecs;
 pub(crate) mod common;
+mod convert_config;
 pub mod encoding_transcode;
 pub mod enrichment_tables;
 #[cfg(feature = "gcp")]