improve readme

Documentation.md

@@ -2,8 +2,6 @@
 
 ## Commands
 
-Check out the [Documentation](Documentation) for a detailed description.
-
 ### Build
 
 The `build` command analyzes all `.sql` files in the subdirectories of the current directory and creates the corresponding Iceberg Materialized Views in the catalog.

README.md

@@ -24,9 +24,27 @@ During the workflow execution Argo starts Docker containers that run Datafusion
 ## Examples
 
 - [Postgres example](https://killercoda.com/dashbook/scenario/dashtool-postgres)
+- [Mysql example](https://killercoda.com/dashbook/scenario/dashtool-mysql)
+- [Kafka example](https://killercoda.com/dashbook/scenario/dashtool-kafka)
 
 ## Usage
 
+Dashtool goes through a build, a workflow and an apply step to turn the declarative input files into an automatically refreshing data pipeline. This is shown in the following diagram:
+
+```mermaid
+graph LR
+    git[Files in 
+    git repo]
+    catalog[Iceberg tables 
+    in catalog]
+    workflows[Argo
+    workflow]
+    data[Data]
+    git -->|dashtool build|catalog
+    catalog -->|dashtool workflow|workflows
+    workflows -->|dashtool apply|data
+```
+
 Check out the [Documentation](Documentation.md) for a detailed description.
 
 ### Build
@@ -37,20 +55,20 @@ The `build` command analyzes all `.sql` files in the subdirectories of the curre
 dashtool build
 ```
 
-### Create Workflow
+### Workflow
 
 The `workflow` command creates a lineage DAG based on the `.sql` files and constructs an Argo workflow based on it. It stores the Workflow configuration file in `argo/workflow.yaml`.
 
 ```shell
 dashtool workflow
 ```
 
-### Apply Workflow to the Kubernetes cluster
+### Apply
 
 To apply the latest version of the workflow to the Kubernetes cluster run the following command:
 
 ```shell
-kubectl apply -f argo/workflow.yaml
+dashtool apply
 ```
 
 ## Installation

dashtool/src/main.rs

@@ -1,11 +1,11 @@
-use std::{fs, str::FromStr, sync::Arc};
+use std::{fs, process::Command, str::FromStr, sync::Arc};
 
 use anyhow::anyhow;
 use dashtool::{
     build::build,
     error::Error,
     plugins::{sql::SqlPlugin, Config, Plugin},
-    workflow::{workflow, WORKFLOW_DIR},
+    workflow::workflow,
 };
 
 use clap::{Parser, Subcommand};
@@ -15,20 +15,35 @@ use clap::{Parser, Subcommand};
 struct Args {
     #[command(subcommand)]
     commands: Commands,
+    /// Set file for Argo workflow definition
+    #[arg(short, long)]
+    file: Option<String>,
 }
 
 #[derive(Subcommand)]
 enum Commands {
     Build,
     Workflow,
+    Apply,
 }
 
+static DASHTOOL_CONFIG: &str = "dashtool.json";
+static OUTPUT_FILE: &str = "argo/workflow.yaml";
+static DAG_DIR: &str = ".dashtool/dags";
+
 #[tokio::main]
 async fn main() -> Result<(), Error> {
     let args = Args::parse();
 
-    fs::create_dir_all(".dashtool/dags").ok();
-    fs::create_dir_all(WORKFLOW_DIR).ok();
+    let output = args.file.unwrap_or(OUTPUT_FILE.to_owned());
+
+    fs::create_dir_all(DAG_DIR).ok();
+    fs::create_dir_all(
+        std::path::Path::new(&output)
+            .parent()
+            .ok_or(Error::Anyhow(anyhow!("Output file cannot be a directory")))?,
+    )
+    .ok();
     #[cfg(not(target_arch = "wasm32"))]
     fs::create_dir_all(
         dirs::config_local_dir()
@@ -37,7 +52,7 @@ async fn main() -> Result<(), Error> {
             + "/dashtool",
     )?;
 
-    let config_json = fs::read_to_string("dashtool.json")?;
+    let config_json = fs::read_to_string(DASHTOOL_CONFIG)?;
     let config: Config = serde_json::from_str(&shellexpand::env(&config_json)?)?;
 
     let plugin: Arc<dyn Plugin> = match config {
@@ -48,6 +63,39 @@ async fn main() -> Result<(), Error> {
 
     match args.commands {
         Commands::Build => build(plugin).await,
-        Commands::Workflow => workflow(plugin),
+        Commands::Workflow => workflow(plugin, &output),
+        Commands::Apply => {
+            if cfg!(target_os = "windows") {
+                Command::new("kubectl")
+                    .args(["apply", "-f", &output])
+                    .output()
+                    .and_then(|x| {
+                        if x.status.success() {
+                            Ok(())
+                        } else {
+                            Err(std::io::Error::new(
+                                std::io::ErrorKind::Other,
+                                "Failed to apply Argo workflow to Kubernetes cluster",
+                            ))
+                        }
+                    })
+                    .map_err(Error::from)
+            } else {
+                Command::new("kubectl")
+                    .args(["apply", "-f", &output])
+                    .output()
+                    .and_then(|x| {
+                        if x.status.success() {
+                            Ok(())
+                        } else {
+                            Err(std::io::Error::new(
+                                std::io::ErrorKind::Other,
+                                "Failed to apply Argo workflow to Kubernetes cluster",
+                            ))
+                        }
+                    })
+                    .map_err(Error::from)
+            }
+        }
     }
 }

dashtool/src/workflow/mod.rs

@@ -24,9 +24,8 @@ use self::template::{iceberg_template, singer_template};
 mod template;
 
 static API_VERSION: &str = "argoproj.io/v1alpha1";
-pub static WORKFLOW_DIR: &str = "argo";
 
-pub fn workflow(plugin: Arc<dyn Plugin>) -> Result<(), Error> {
+pub fn workflow(plugin: Arc<dyn Plugin>, output: &str) -> Result<(), Error> {
     let repo = gix::discover(".")?;
     let branch = branch(&repo)?;
 
@@ -200,10 +199,7 @@ pub fn workflow(plugin: Arc<dyn Plugin>) -> Result<(), Error> {
         workflow_yaml.push_str(&yaml);
     }
 
-    fs::write(
-        &(WORKFLOW_DIR.to_string() + "/workflow.yaml"),
-        workflow_yaml,
-    )?;
+    fs::write(output, workflow_yaml)?;
 
     Ok(())
 }