fix: remove obsolete resources (#1543)

* wip: remove resources from workspace

* cmt

* wip: migrate out old resources in deployer and r-r

* revert: unrelated changes

* feat(deployer): ignore invalid resources from load response instead of panic

* delete custom and protect r-r from invalid resource types

* fix some ci

* comment

* feat: delete old logs from deployer state

* fix: i'm good at sql

* fix: re-add persist resource

* clippy

* nit: add back test for string mappings

common/Cargo.toml

@@ -92,6 +92,7 @@ claims = [
 display = ["chrono/clock", "comfy-table", "crossterm"]
 models = ["async-trait", "http", "reqwest", "service", "thiserror"]
 persist = ["sqlx", "rand"]
+sqlx = ["dep:sqlx", "sqlx/sqlite"]
 service = ["chrono/serde", "display", "tracing", "tracing-subscriber", "uuid"]
 test-utils = ["wiremock"]
 tracing = ["dep:tracing"]

common/src/models/resource.rs

@@ -30,10 +30,7 @@ pub fn get_resource_tables(
             let title = match x.r#type {
                 Type::Database(_) => "Databases",
                 Type::Secrets => "Secrets",
-                Type::StaticFolder => "Static Folder",
                 Type::Persist => "Persist",
-                Type::Turso => "Turso",
-                Type::Metadata => "Metadata",
                 Type::Custom => "Custom",
             };
 
@@ -58,16 +55,8 @@ pub fn get_resource_tables(
             output.push(get_secrets_table(secrets, service_name, raw));
         };
 
-        if let Some(static_folders) = resource_groups.get("Static Folder") {
-            output.push(get_static_folder_table(static_folders, service_name, raw));
-        };
-
-        if let Some(persist) = resource_groups.get("Persist") {
-            output.push(get_persist_table(persist, service_name, raw));
-        };
-
-        if let Some(custom) = resource_groups.get("Custom") {
-            output.push(get_custom_resources_table(custom, service_name, raw));
+        if resource_groups.get("Persist").is_some() {
+            output.push(format!("This persist instance is linked to {service_name}\nShuttle Persist: {service_name}\n"));
         };
 
         output.join("\n")
@@ -171,81 +160,3 @@ fn get_secrets_table(secrets: &[&Response], service_name: &str, raw: bool) -> St
 
     format!("These secrets can be accessed by {service_name}\n{table}\n")
 }
-
-fn get_static_folder_table(static_folders: &[&Response], service_name: &str, raw: bool) -> String {
-    let mut table = Table::new();
-
-    if raw {
-        table
-            .load_preset(NOTHING)
-            .set_header(vec![Cell::new("Folders").set_alignment(CellAlignment::Left)]);
-    } else {
-        table
-            .load_preset(UTF8_FULL)
-            .apply_modifier(UTF8_ROUND_CORNERS)
-            .set_header(vec![Cell::new("Folders")
-                .set_alignment(CellAlignment::Center)
-                .add_attribute(Attribute::Bold)]);
-    }
-
-    for folder in static_folders {
-        let path = serde_json::from_value::<String>(folder.config.clone()).unwrap();
-
-        table.add_row(vec![path]);
-    }
-
-    format!("These static folders can be accessed by {service_name}\n{table}\n")
-}
-
-fn get_persist_table(persist_instances: &[&Response], service_name: &str, raw: bool) -> String {
-    let mut table = Table::new();
-
-    if raw {
-        table.load_preset(NOTHING).set_header(vec![
-            Cell::new("Instances").set_alignment(CellAlignment::Left)
-        ]);
-    } else {
-        table
-            .load_preset(UTF8_FULL)
-            .apply_modifier(UTF8_ROUND_CORNERS)
-            .set_header(vec![Cell::new("Instances")
-                .set_alignment(CellAlignment::Center)
-                .add_attribute(Attribute::Bold)]);
-    }
-
-    for _ in persist_instances {
-        table.add_row(vec!["Instance"]);
-    }
-
-    format!("These persist instances are linked to {service_name}\n{table}\n")
-}
-
-fn get_custom_resources_table(
-    custom_resource_instances: &[&Response],
-    service_name: &str,
-    raw: bool,
-) -> String {
-    let mut table = Table::new();
-
-    if raw {
-        table.load_preset(NOTHING).set_header(vec![
-            Cell::new("Instances").set_alignment(CellAlignment::Left)
-        ]);
-    } else {
-        table
-            .load_preset(UTF8_FULL)
-            .apply_modifier(UTF8_ROUND_CORNERS)
-            .set_header(vec![Cell::new("Instances")
-                .set_alignment(CellAlignment::Center)
-                .add_attribute(Attribute::Bold)]);
-    }
-
-    for (idx, _) in custom_resource_instances.iter().enumerate() {
-        // TODO: add some information that would make the custom resources identifiable.
-        // This requires changing the backend resource list response to include a resource identifier
-        // that can be used to query for more info related to a resource.
-        table.add_row(vec![format!("custom-resource-{}", idx.to_string())]);
-    }
-
-    format!("These custom resource instances are linked to {service_name}\n{table}\n")
-}

common/src/resource.rs

@@ -18,15 +18,26 @@ pub struct Response {
     pub data: Value,
 }
 
+impl Response {
+    pub fn into_bytes(self) -> Vec<u8> {
+        self.to_bytes()
+    }
+
+    pub fn to_bytes(&self) -> Vec<u8> {
+        serde_json::to_vec(self).expect("to turn resource into a vec")
+    }
+
+    pub fn from_bytes(bytes: Vec<u8>) -> Self {
+        serde_json::from_slice(&bytes).expect("to turn bytes into a resource")
+    }
+}
+
 #[derive(Clone, Copy, Debug, Deserialize, Serialize, Eq, PartialEq)]
 #[serde(rename_all = "lowercase")]
 pub enum Type {
     Database(database::Type),
     Secrets,
-    StaticFolder,
     Persist,
-    Turso,
-    Metadata,
     Custom,
 }
 
@@ -42,41 +53,83 @@ impl FromStr for Type {
         } else {
             match s {
                 "secrets" => Ok(Self::Secrets),
-                "static_folder" => Ok(Self::StaticFolder),
-                "metadata" => Ok(Self::Metadata),
                 "persist" => Ok(Self::Persist),
-                "turso" => Ok(Self::Turso),
                 "custom" => Ok(Self::Custom),
                 _ => Err(format!("'{s}' is an unknown resource type")),
             }
         }
     }
 }
 
-impl Response {
-    pub fn into_bytes(self) -> Vec<u8> {
-        self.to_bytes()
-    }
-
-    pub fn to_bytes(&self) -> Vec<u8> {
-        serde_json::to_vec(self).expect("to turn resource into a vec")
-    }
-
-    pub fn from_bytes(bytes: Vec<u8>) -> Self {
-        serde_json::from_slice(&bytes).expect("to turn bytes into a resource")
-    }
-}
-
 impl Display for Type {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Type::Database(db_type) => write!(f, "database::{db_type}"),
             Type::Secrets => write!(f, "secrets"),
-            Type::StaticFolder => write!(f, "static_folder"),
             Type::Persist => write!(f, "persist"),
-            Type::Turso => write!(f, "turso"),
-            Type::Metadata => write!(f, "metadata"),
             Type::Custom => write!(f, "custom"),
         }
     }
 }
+
+// this can be removed when deployers AND r-r no longer hold resources in sqlite state
+#[cfg(feature = "sqlx")]
+mod _sqlx {
+    use std::{borrow::Cow, str::FromStr};
+
+    use sqlx::{
+        sqlite::{SqliteArgumentValue, SqliteValueRef},
+        Database, Sqlite,
+    };
+
+    use super::Type;
+
+    impl<DB: Database> sqlx::Type<DB> for Type
+    where
+        str: sqlx::Type<DB>,
+    {
+        fn type_info() -> <DB as Database>::TypeInfo {
+            <str as sqlx::Type<DB>>::type_info()
+        }
+    }
+
+    impl<'q> sqlx::Encode<'q, Sqlite> for Type {
+        fn encode_by_ref(&self, args: &mut Vec<SqliteArgumentValue<'q>>) -> sqlx::encode::IsNull {
+            args.push(SqliteArgumentValue::Text(Cow::Owned(self.to_string())));
+
+            sqlx::encode::IsNull::No
+        }
+    }
+
+    impl<'r> sqlx::Decode<'r, Sqlite> for Type {
+        fn decode(value: SqliteValueRef<'r>) -> Result<Self, sqlx::error::BoxDynError> {
+            let value = <&str as sqlx::Decode<Sqlite>>::decode(value)?;
+
+            Self::from_str(value).map_err(Into::into)
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn to_string_and_back() {
+        let inputs = [
+            Type::Database(database::Type::AwsRds(database::AwsRdsEngine::Postgres)),
+            Type::Database(database::Type::AwsRds(database::AwsRdsEngine::MySql)),
+            Type::Database(database::Type::AwsRds(database::AwsRdsEngine::MariaDB)),
+            Type::Database(database::Type::Shared(database::SharedEngine::Postgres)),
+            Type::Database(database::Type::Shared(database::SharedEngine::MongoDb)),
+            Type::Secrets,
+            Type::Persist,
+            Type::Custom,
+        ];
+
+        for input in inputs {
+            let actual = Type::from_str(&input.to_string()).unwrap();
+            assert_eq!(input, actual, ":{} should map back to itself", input);
+        }
+    }
+}

deployer/Cargo.toml

@@ -6,7 +6,7 @@ license.workspace = true
 description = "Service with instances created per project for handling the compilation, loading, and execution of Shuttle services"
 
 [dependencies]
-shuttle-common = { workspace = true, features = ["backend", "models"] }
+shuttle-common = { workspace = true, features = ["backend", "models", "sqlx"] }
 shuttle-proto = { workspace = true, features = ["resource-recorder"] }
 shuttle-service = { workspace = true, features = ["builder", "runner"] }
 

deployer/migrations/0006_delete_outdated_resources.sql

@@ -0,0 +1,5 @@
+-- delete any old records of these so that they are not sent to r-r
+DELETE FROM resources WHERE type = "static_folder";
+DELETE FROM resources WHERE type = "turso";
+DELETE FROM resources WHERE type = "metadata";
+DELETE FROM resources WHERE type = "custom";

deployer/migrations/0007_delete_logs.sql

@@ -0,0 +1,2 @@
+-- Logs in deployers only exist for getting pre-logger logs, which are now deadweight
+DROP TABLE IF EXISTS logs;

deployer/src/deployment/run.rs

@@ -381,13 +381,15 @@ async fn load(
             let resources = response
                 .resources
                 .into_iter()
-                .map(|res| {
-                    let resource: resource::Response = serde_json::from_slice(&res).unwrap();
-                    record_request::Resource {
-                        r#type: resource.r#type.to_string(),
-                        config: resource.config.to_string().into_bytes(),
-                        data: resource.data.to_string().into_bytes(),
-                    }
+                .filter_map(|res| {
+                    // filter out resources with invalid types
+                    serde_json::from_slice::<resource::Response>(&res)
+                        .ok()
+                        .map(|r| record_request::Resource {
+                            r#type: r.r#type.to_string(),
+                            config: r.config.to_string().into_bytes(),
+                            data: r.data.to_string().into_bytes(),
+                        })
                 })
                 .collect();
             resource_manager