uses sqlx query instead of migrations for creating extensions

tembo-cli/Cargo.toml

@@ -16,6 +16,10 @@ repository = "https://github.com/tembo-io/tembo-cli"
 name = "tembo"
 path = "src/main.rs"
 
+[lib]
+name = "tembo"
+path = "src/lib.rs"
+
 [dependencies]
 anyhow = "1.0.71"
 clap = { version = "4.3.12", features = ["cargo", "wrap_help", "derive"] }

tembo-cli/README.md

@@ -28,7 +28,6 @@ cargo install tembo-cli
 The `tembo init` command initializes your environment with following files. Run init in the directory you want to create the `tembo.toml` file.
 
 * `tembo.toml` example configuration file
-* `migrations` directory for sql migrations
 * `~/.tembo/context` file with various contexts user can connect to
 * `~/.tembo/credentials` file with credentials & api urls
 
@@ -51,7 +50,7 @@ Validates `tembo.toml` and other configurations files.
 
 #### `tembo apply`
 
-Validates tembo.toml (same as `tembo validate`) and applies the changes to the context selected. It applies changes and runs migration for all databases.
+Validates tembo.toml (same as `tembo validate`) and applies the changes to the context selected.
 
 ##### Environment:
 
@@ -60,7 +59,6 @@ Validates tembo.toml (same as `tembo validate`) and applies the changes to the c
     * generates `Dockerfile` for each instance & builds a docker image
     * generates `docker-compose` to provision all instances
     * runs `docker-compose up -d` to spin up all instances
-    * runs `sqlx migration` against the instances
 
   * ###### Tembo-Cloud: 
     * Creates/updates instance on tembo-cloud by calling the api against the appropriate environment

tembo-cli/src/cli/sqlx_utils.rs

@@ -2,6 +2,7 @@ use crate::tui::colors;
 use colorful::{Color, Colorful};
 use spinoff::{spinners, Spinner};
 use sqlx::migrate::Migrator;
+use sqlx::postgres::PgConnectOptions;
 use sqlx::Pool;
 use sqlx::Postgres;
 use std::path::Path;
@@ -10,38 +11,16 @@ use temboclient::models::ConnectionInfo;
 pub struct SqlxUtils {}
 
 impl SqlxUtils {
-    // run sqlx migrate
-    pub async fn run_migrations(
-        connection_info: ConnectionInfo,
-        instance_name: String,
-    ) -> Result<(), anyhow::Error> {
-        let mut sp = Spinner::new(
-            spinners::Dots,
-            "Running SQL migration",
-            spinoff::Color::White,
-        );
+    pub async fn execute_sql(instance_name: String, sql: String) -> Result<(), anyhow::Error> {
+        let connect_options = PgConnectOptions::new()
+            .username("postgres")
+            .password("postgres")
+            .host(&format!("{}.local.tembo.io", instance_name))
+            .database("postgres");
 
-        let connection_string = format!(
-            "postgresql://{}:{}@{}:{}",
-            connection_info.user,
-            connection_info.password,
-            connection_info.host,
-            connection_info.port
-        );
+        let pool = sqlx::PgPool::connect_with(connect_options).await?;
 
-        let pool = Pool::<Postgres>::connect(connection_string.as_str()).await?;
-
-        let path = instance_name.clone() + "/migrations";
-        let m = Migrator::new(Path::new(&path)).await?;
-        m.run(&pool).await?;
-
-        sp.stop_with_message(&format!(
-            "{} {}",
-            "✓".color(colors::indicator_good()).bold(),
-            format!("SQL migration completed for {}", instance_name)
-                .color(Color::White)
-                .bold()
-        ));
+        sqlx::query(&sql).execute(&pool).await?;
 
         Ok(())
     }

tembo-cli/src/cli/tembo_config.rs

@@ -1,4 +1,5 @@
 use controller::app_service::types::AppService;
+use controller::extensions::types::Extension as ControllerExtension;
 use serde::{Deserialize, Deserializer, Serialize};
 use std::collections::HashMap;
 use tembo_stacks::apps::types::AppType;
@@ -35,6 +36,7 @@ pub struct InstanceSettings {
     pub extensions: Option<HashMap<String, Extension>>,
     pub app_services: Option<Vec<AppType>>,
     pub controller_app_services: Option<HashMap<String, AppService>>,
+    pub final_extensions: Option<Vec<ControllerExtension>>,
     pub extra_domains_rw: Option<Vec<String>>,
     pub ip_allow_list: Option<Vec<String>>,
 }

tembo-cli/src/cmd/apply.rs

@@ -9,6 +9,8 @@ use controller::extensions::types::TrunkInstall as ControllerTrunkInstall;
 use controller::stacks::get_stack;
 use controller::stacks::types::StackType as ControllerStackType;
 use log::info;
+use spinoff::spinners;
+use spinoff::Spinner;
 use std::path::Path;
 use std::{
     collections::HashMap,
@@ -128,28 +130,18 @@ fn docker_apply(
 
     Docker::docker_compose_down(false)?;
 
-    for (_key, instance_setting) in instance_settings.iter_mut() {
-        let result = docker_apply_instance(verbose, instance_setting);
+    let mut final_instance_settings: HashMap<String, InstanceSettings> = Default::default();
 
-        let app_svc = match result {
-            Ok(i) => i,
-            Err(error) => {
-                tui::error(&format!("Error creating instance: {}", error));
-                return Ok(());
-            }
-        };
-
-        if app_svc.is_some() {
-            let mut controller_app_svcs: HashMap<String, AppService> = Default::default();
-            for cas in app_svc.unwrap().iter() {
-                controller_app_svcs.insert(cas.name.clone(), cas.to_owned());
-            }
-
-            instance_setting.controller_app_services = Some(controller_app_svcs);
-        }
+    for (_key, instance_setting) in instance_settings.iter_mut() {
+        let final_instance_setting = docker_apply_instance(verbose, instance_setting.to_owned())?;
+        final_instance_settings.insert(
+            final_instance_setting.instance_name.clone(),
+            final_instance_setting,
+        );
     }
 
-    let rendered_dockercompose: String = get_rendered_dockercompose(instance_settings.clone())?;
+    let rendered_dockercompose: String =
+        get_rendered_dockercompose(final_instance_settings.clone())?;
 
     FileUtils::create_file(
         DOCKERCOMPOSE_NAME.to_string(),
@@ -160,23 +152,32 @@ fn docker_apply(
 
     Docker::docker_compose_up(verbose)?;
 
-    // Allows DB instance to be ready before running migrations
+    // Allows DB instance to be ready before running CREATE EXTENSION script
     sleep(Duration::from_secs(5));
 
     let port = 5432;
 
-    for (_key, instance_setting) in instance_settings.clone().iter() {
-        let conn_info = ConnectionInfo {
-            host: format!("{}.local.tembo.io", instance_setting.instance_name).to_owned(),
-            pooler_host: Some(Some("localhost-pooler".to_string())),
-            port,
-            user: "postgres".to_owned(),
-            password: "postgres".to_owned(),
-        };
-        Runtime::new().unwrap().block_on(SqlxUtils::run_migrations(
-            conn_info,
-            instance_setting.instance_name.clone(),
-        ))?;
+    for (_key, instance_setting) in final_instance_settings.clone().iter() {
+        let instance_name = &instance_setting.instance_name;
+
+        let mut sp = Spinner::new(spinners::Dots, "Creating extensions", spinoff::Color::White);
+
+        for ext in instance_setting.final_extensions.clone().unwrap().iter() {
+            let query = &format!("CREATE EXTENSION IF NOT EXISTS {} CASCADE", ext.name);
+
+            Runtime::new().unwrap().block_on(SqlxUtils::execute_sql(
+                instance_name.to_string(),
+                query.to_string(),
+            ))?;
+        }
+
+        sp.stop_with_message(&format!(
+            "{} {}",
+            "✓".color(colors::indicator_good()).bold(),
+            format!("Extensions created for instance {}", instance_name)
+                .color(colorful::Color::White)
+                .bold()
+        ));
 
         // If all of the above was successful, we can print the url to user
         instance_started(
@@ -188,33 +189,32 @@ fn docker_apply(
             "local",
         );
     }
-
     Ok(())
 }
 
 fn docker_apply_instance(
     verbose: bool,
-    instance_settings: &InstanceSettings,
-) -> Result<Option<Vec<AppService>>, anyhow::Error> {
+    mut instance_setting: InstanceSettings,
+) -> Result<InstanceSettings, anyhow::Error> {
     FileUtils::create_dir(
-        instance_settings.instance_name.clone(),
-        instance_settings.instance_name.clone(),
+        instance_setting.instance_name.clone(),
+        instance_setting.instance_name.clone(),
     )?;
 
-    let stack_type = ControllerStackType::from_str(instance_settings.stack_type.as_str())
+    let stack_type = ControllerStackType::from_str(instance_setting.stack_type.as_str())
         .unwrap_or(ControllerStackType::Standard);
     let stack = get_stack(stack_type);
 
     let extensions = merge_options(
         stack.extensions.clone(),
         Some(get_extensions_controller(
-            instance_settings.extensions.clone(),
+            instance_setting.extensions.clone(),
         )),
     );
     let trunk_installs = merge_options(
         stack.trunk_installs.clone(),
         Some(get_trunk_installs_controller(
-            instance_settings.extensions.clone(),
+            instance_setting.extensions.clone(),
         )),
     );
 
@@ -224,7 +224,7 @@ fn docker_apply_instance(
         app_services,
         pg_configs: _,
     } = merge_app_reqs(
-        instance_settings.app_services.clone(),
+        instance_setting.app_services.clone(),
         stack.app_services.clone(),
         extensions,
         trunk_installs,
@@ -235,30 +235,32 @@ fn docker_apply_instance(
 
     FileUtils::create_file(
         DOCKERFILE_NAME.to_string(),
-        instance_settings.instance_name.clone() + "/" + DOCKERFILE_NAME,
+        instance_setting.instance_name.clone() + "/" + DOCKERFILE_NAME,
         rendered_dockerfile,
         true,
     )?;
 
-    let rendered_migrations: String = get_rendered_migrations_file(&extensions)?;
-
-    FileUtils::create_file(
-        "extensions".to_string(),
-        instance_settings.instance_name.clone() + "/" + "migrations/1_extensions.sql",
-        rendered_migrations,
-        true,
-    )?;
+    instance_setting.final_extensions = extensions;
 
     FileUtils::create_file(
         POSTGRESCONF_NAME.to_string(),
-        instance_settings.instance_name.clone() + "/" + POSTGRESCONF_NAME,
-        get_postgres_config(instance_settings),
+        instance_setting.instance_name.clone() + "/" + POSTGRESCONF_NAME,
+        get_postgres_config(&instance_setting),
         true,
     )?;
 
-    Docker::build(instance_settings.instance_name.clone(), verbose)?;
+    Docker::build(instance_setting.instance_name.clone(), verbose)?;
 
-    Ok(app_services)
+    if app_services.is_some() {
+        let mut controller_app_svcs: HashMap<String, AppService> = Default::default();
+        for cas in app_services.unwrap().iter() {
+            controller_app_svcs.insert(cas.name.clone(), cas.to_owned());
+        }
+
+        instance_setting.controller_app_services = Some(controller_app_svcs);
+    }
+
+    Ok(instance_setting)
 }
 
 pub fn tembo_cloud_apply_instance(
@@ -309,13 +311,6 @@ pub fn tembo_cloud_apply_instance(
                 env.clone(),
             )?;
 
-            if Path::new(&instance_settings.instance_name).exists() {
-                Runtime::new().unwrap().block_on(SqlxUtils::run_migrations(
-                    conn_info.clone(),
-                    instance_settings.instance_name.clone(),
-                ))?;
-            }
-
             // If all of the above was successful we can stop the spinner and show a success message
             sp.stop_with_message(&format!(
                 "{} {}",
@@ -651,6 +646,7 @@ fn merge_settings(base: &InstanceSettings, overlay: OverlayInstanceSettings) ->
             .postgres_configurations
             .or_else(|| base.postgres_configurations.clone()),
         extensions: overlay.extensions.or_else(|| base.extensions.clone()),
+        final_extensions: None,
         app_services: None,
         controller_app_services: None,
         extra_domains_rw: overlay
@@ -753,28 +749,6 @@ pub fn get_rendered_dockerfile(
     Ok(rendered_dockerfile)
 }
 
-pub fn get_rendered_migrations_file(
-    extensions: &Option<Vec<ControllerExtension>>,
-) -> Result<String, anyhow::Error> {
-    // Include the migrations template directly into the binary
-    let contents = include_str!("../../tembo/migrations.sql.template");
-
-    let mut tera = Tera::new("templates/**/*")
-        .map_err(|e| anyhow::anyhow!("Error initializing Tera: {}", e))?;
-    tera.add_raw_template("migrations", contents)
-        .map_err(|e| anyhow::anyhow!("Error adding raw template: {}", e))?;
-
-    let mut context = Context::new();
-
-    context.insert("extensions", extensions);
-
-    let rendered_migrations = tera
-        .render("migrations", &context)
-        .map_err(|e| anyhow::anyhow!("Error rendering template: {}", e))?;
-
-    Ok(rendered_migrations)
-}
-
 fn get_postgres_config(instance_settings: &InstanceSettings) -> String {
     let mut postgres_config = String::from("");
     let qoute_new_line = "\'\n";

tembo-cli/src/cmd/init.rs

@@ -48,13 +48,6 @@ pub fn execute() -> Result<(), anyhow::Error> {
 
     FileUtils::download_file(filepath, filename, false)?;
 
-    match FileUtils::create_dir("migrations directory".to_string(), "migrations".to_string()) {
-        Ok(t) => t,
-        Err(e) => {
-            return Err(e);
-        }
-    }
-
     confirmation("Tembo initialized successfully!");
 
     Ok(())

tembo-cli/src/lib.rs

@@ -0,0 +1,2 @@
+pub mod cli;
+pub mod tui;