From c99df6b8023e376a702051362149943f7542f984 Mon Sep 17 00:00:00 2001 From: HttpRafa <60099368+HttpRafa@users.noreply.github.com> Date: Sat, 11 Jan 2025 13:01:29 +0100 Subject: [PATCH 01/21] feat: Seperate wasmtime and wasm driver configurations --- controller/src/application/driver/wasm.rs | 22 ++++++++++++++++----- docs/controller/drivers/wasm/permissions.md | 8 +------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/controller/src/application/driver/wasm.rs b/controller/src/application/driver/wasm.rs index f3fbb629..1155d0fb 100644 --- a/controller/src/application/driver/wasm.rs +++ b/controller/src/application/driver/wasm.rs @@ -38,13 +38,13 @@ const WASM_DIRECTORY: &str = "wasm"; /* Caching of compiled wasm artifacts and other configuration */ const CONFIG_FILE: &str = "wasm.toml"; -const DEFAULT_CONFIG: &str = r#"# For more settings, please refer to the documentation: +const ENGINE_CONFIG_FILE: &str = "wasm-engine.toml"; +const DEFAULT_ENGINE_CONFIG: &str = r#"# For more settings, please refer to the documentation: # https://bytecodealliance.github.io/wasmtime/cli-cache.html [cache] -enabled = true - -# This section is crucial for granting the drivers their required permissions +enabled = true"#; +const DEFAULT_CONFIG: &str = r#"# This configuration is crucial for granting the drivers their required permissions # https://httprafa.github.io/atomic-cloud/controller/drivers/wasm/permissions/ [[drivers]] @@ -187,7 +187,7 @@ impl WasmDriver { let mut engine_config = Config::new(); engine_config.wasm_component_model(true); if let Err(error) = - engine_config.cache_config_load(Storage::get_configs_folder().join(CONFIG_FILE)) + engine_config.cache_config_load(Storage::get_configs_folder().join(ENGINE_CONFIG_FILE)) { warn!( "Failed to enable caching for wasmtime engine: {}", @@ -274,6 +274,17 @@ impl WasmDriver { drivers: &mut Vec>, ) -> WasmConfig { // Check if cache configuration exists + { + let engine_config_file = Storage::get_configs_folder().join(ENGINE_CONFIG_FILE); + if !engine_config_file.exists() { + fs::write(&engine_config_file, DEFAULT_ENGINE_CONFIG).unwrap_or_else(|error| { + warn!( + "Failed to create default wasmtime configuration file: {}", + &error + ) + }); + } + } let config_file = Storage::get_configs_folder().join(CONFIG_FILE); if !config_file.exists() { fs::write(&config_file, DEFAULT_CONFIG).unwrap_or_else(|error| { @@ -283,6 +294,7 @@ impl WasmDriver { ) }); } + let config = WasmConfig::load_from_file(&config_file).unwrap_or_else(|error| { warn!( "Failed to load wasm configuration file: {}", diff --git a/docs/controller/drivers/wasm/permissions.md b/docs/controller/drivers/wasm/permissions.md index 5b1c2a4a..d843cb67 100644 --- a/docs/controller/drivers/wasm/permissions.md +++ b/docs/controller/drivers/wasm/permissions.md @@ -9,13 +9,7 @@ nano wasm.toml ``` ```toml -# For more settings, please refer to the documentation: -# https://bytecodealliance.github.io/wasmtime/cli-cache.html - -[cache] -enabled = true - -# This section is crucial for granting the drivers their required permissions +# This configuration is crucial for granting the drivers their required permissions # https://httprafa.github.io/atomic-cloud/controller/drivers/wasm/permissions/ [[drivers]] From 092de8e0cb8689d125dffabb5475bc1288754ecc Mon Sep 17 00:00:00 2001 From: HttpRafa <60099368+HttpRafa@users.noreply.github.com> Date: Sat, 11 Jan 2025 13:23:00 +0100 Subject: [PATCH 02/21] feat: Add empty local driver --- .github/workflows/release.yml | 5 +- Cargo.toml | 1 + Makefile | 2 +- drivers/local/Cargo.toml | 24 +++++++++ drivers/local/build.rs | 74 ++++++++++++++++++++++++++++ drivers/local/src/config.rs | 25 ++++++++++ drivers/local/src/driver.rs | 41 +++++++++++++++ drivers/local/src/driver/cloudlet.rs | 36 ++++++++++++++ drivers/local/src/log.rs | 27 ++++++++++ drivers/local/src/main.rs | 24 +++++++++ 10 files changed, 257 insertions(+), 2 deletions(-) create mode 100644 drivers/local/Cargo.toml create mode 100644 drivers/local/build.rs create mode 100644 drivers/local/src/config.rs create mode 100644 drivers/local/src/driver.rs create mode 100644 drivers/local/src/driver/cloudlet.rs create mode 100644 drivers/local/src/log.rs create mode 100644 drivers/local/src/main.rs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 716bd3de..36f3c19b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,11 +72,12 @@ jobs: git push -u origin $BRANCH sed -i 's/client_version=0\.0\.0-nightly/client_version=\${REF:11}/g' clients/jvm/gradle.properties sed -i 's/version = "0\.0\.0-nightly"/version = "\${REF:11}"/g' drivers/pterodactyl/Cargo.toml + sed -i 's/version = "0\.0\.0-nightly"/version = "\${REF:11}"/g' drivers/local/Cargo.toml sed -i 's/version = "0\.0\.0-nightly"/version = "\${REF:11}"/g' clients/wrapper/Cargo.toml sed -i 's/version = "0\.0\.0-nightly"/version = "\${REF:11}"/g' controller/Cargo.toml sed -i 's/version = "0\.0\.0-nightly"/version = "\${REF:11}"/g' common/Cargo.toml sed -i 's/version = "0\.0\.0-nightly"/version = "\${REF:11}"/g' cli/Cargo.toml - git add clients/jvm/gradle.properties drivers/pterodactyl/Cargo.toml clients/wrapper/Cargo.toml controller/Cargo.toml common/Cargo.toml cli/Cargo.toml + git add clients/jvm/gradle.properties drivers/pterodactyl/Cargo.toml drivers/local/Cargo.toml clients/wrapper/Cargo.toml controller/Cargo.toml common/Cargo.toml cli/Cargo.toml git commit -m "ci(release): bump version" git push @@ -112,6 +113,7 @@ jobs: cp ./target/x86_64-pc-windows-gnu/release/wrapper.exe wrapper-windows-x86_64.exe cp ./target/wasm32-wasip2/release/pterodactyl.wasm pterodactyl-driver.wasm + cp ./target/wasm32-wasip2/release/local.wasm local-driver.wasm cp $(find ./clients/jvm/paper/build -name "*-all.jar") paper-client.jar @@ -131,4 +133,5 @@ jobs: wrapper-linux-x86_64 wrapper-windows-x86_64.exe pterodactyl-driver.wasm + local-driver.wasm paper-client.jar \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 29de91ca..61c41295 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ # Drivers "drivers/pterodactyl", + "drivers/local", # Clients "cli", diff --git a/Makefile b/Makefile index 7e3f1156..9cc2e83d 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,6 @@ # Configuration WASM_RUSTFLAGS = -Z wasi-exec-model=reactor WASM_TARGET = wasm32-wasip2 -WASM_COMPONENT = target/wasm32-wasip2/release/pterodactyl.wasm # Directories RUN_DIR = run @@ -72,6 +71,7 @@ build-wrapper: build-drivers: $(SETENV) RUSTFLAGS="$(WASM_RUSTFLAGS)" cargo build -p pterodactyl --target $(WASM_TARGET) --release + cargo build -p local --target $(WASM_TARGET) --release # Create driver directory if it doesn't exist $(DRIVER_DIR): diff --git a/drivers/local/Cargo.toml b/drivers/local/Cargo.toml new file mode 100644 index 00000000..6beb1fe1 --- /dev/null +++ b/drivers/local/Cargo.toml @@ -0,0 +1,24 @@ +cargo-features = ["per-package-target"] + +[package] +name = "local" +version = "0.0.0-nightly" +edition = "2021" +forced-target = "wasm32-wasip2" + +[dependencies] +# Common +common = { path = "../../common" } + +# Wasm plugin +wit-bindgen = "0.37.0" + +# Error handling +anyhow = "1.0.95" + +# Configuration +serde = { version = "1.0.217", features = ["derive"] } +toml = "0.8.19" + +[build-dependencies] +toml = "0.8.19" \ No newline at end of file diff --git a/drivers/local/build.rs b/drivers/local/build.rs new file mode 100644 index 00000000..059f4719 --- /dev/null +++ b/drivers/local/build.rs @@ -0,0 +1,74 @@ +use std::{ + env, + fs::{self, File}, + io::Write, +}; + +fn main() -> Result<(), Box> { + generate_build_info(); + Ok(()) +} + +fn generate_build_info() { + let out_dir = env::var("OUT_DIR").unwrap(); + let mut file = File::create(format!("{}/build_info.rs", out_dir)).unwrap(); + + let commit = env::var("CURRENT_COMMIT").unwrap_or_else(|_| "unknown".to_string()); + let build = env::var("CURRENT_BUILD").unwrap_or_else(|_| "0".to_string()); + + let version = get_version_info().expect("Unable to get version information"); + let protocol_version = + get_protocol_version_info().expect("Unable to get protocol version information"); + + writeln!(file, "use common::version::{{Stage, Version}};").unwrap(); + writeln!(file, "pub const VERSION: Version = Version {{").unwrap(); + writeln!(file, " major: {},", version.0).unwrap(); + writeln!(file, " minor: {},", version.1).unwrap(); + writeln!(file, " patch: {},", version.2).unwrap(); + writeln!(file, " build: {},", build).unwrap(); + writeln!(file, " commit: \"{}\",", commit).unwrap(); + writeln!(file, " stage: Stage::{},", version.3).unwrap(); + writeln!(file, " protocol: {},", protocol_version).unwrap(); + writeln!(file, "}};").unwrap(); +} + +fn get_version_info() -> Result<(u16, u16, u16, String), Box> { + let cargo_toml_content = fs::read_to_string("Cargo.toml")?; + let cargo_toml: toml::Value = toml::from_str(&cargo_toml_content)?; + + let version_str = cargo_toml["package"]["version"] + .as_str() + .ok_or("Unable to get version from Cargo.toml")?; + + let version_parts: Vec<&str> = version_str.split('-').collect(); + let version_numbers: Vec = version_parts[0] + .split('.') + .map(|v| v.parse().map_err(|_| "Invalid version part")) + .collect::, _>>()?; + + if version_numbers.len() == 3 { + let stage = if version_parts.len() > 1 { + version_parts[1][0..1].to_uppercase() + &version_parts[1][1..] + } else { + "Stable".to_string() + }; + Ok(( + version_numbers[0], + version_numbers[1], + version_numbers[2], + stage, + )) + } else { + Err("Version must have three parts".into()) + } +} + +fn get_protocol_version_info() -> Result> { + let cargo_toml_content = fs::read_to_string("../../Cargo.toml")?; + let cargo_toml: toml::Value = toml::from_str(&cargo_toml_content)?; + + let value = cargo_toml["workspace"]["metadata"]["protocol-version"] + .as_integer() + .map(|v| v as u32); + value.ok_or("Unable to get protocol version from Cargo.toml".into()) +} diff --git a/drivers/local/src/config.rs b/drivers/local/src/config.rs new file mode 100644 index 00000000..8984b45d --- /dev/null +++ b/drivers/local/src/config.rs @@ -0,0 +1,25 @@ +use anyhow::Result; +use serde::{de::DeserializeOwned, Serialize}; +use std::{fs, path::Path}; + +pub const CONFIG_DIRECTORY: &str = "/configs"; + +pub trait SaveToTomlFile: Serialize { + fn save_to_file(&self, path: &Path, create_parent: bool) -> Result<()> { + if let Some(parent) = path.parent() { + if create_parent { + fs::create_dir_all(parent)?; + } + } + fs::write(path, toml::to_string(self)?)?; + Ok(()) + } +} + +pub trait LoadFromTomlFile: DeserializeOwned { + fn load_from_file(path: &Path) -> Result { + let data = fs::read_to_string(path)?; + let config = toml::from_str(&data)?; + Ok(config) + } +} diff --git a/drivers/local/src/driver.rs b/drivers/local/src/driver.rs new file mode 100644 index 00000000..86c08811 --- /dev/null +++ b/drivers/local/src/driver.rs @@ -0,0 +1,41 @@ +use crate::exports::cloudlet::driver::bridge::{Capabilities, GenericCloudlet, GuestGenericDriver, Information, RemoteController}; + +pub mod cloudlet; + +// Include the build information generated by build.rs +include!(concat!(env!("OUT_DIR"), "/build_info.rs")); + +pub const AUTHORS: [&str; 1] = ["HttpRafa"]; + +pub struct Local { + /* Cloud Identification */ + cloud_identifier: String, +} + +impl GuestGenericDriver for Local { + fn new(cloud_identifier: String) -> Self { + Self { + cloud_identifier, + } + } + + fn init(&self) -> Information { + Information { + authors: AUTHORS.iter().map(|&author| author.to_string()).collect(), + version: VERSION.to_string(), + ready: false, + } + } + + fn init_cloudlet( + &self, + _name: String, + _capabilities: Capabilities, + _controller: RemoteController, + ) -> Result { + Err("Not implemented".to_string()) + } +} + +pub struct LocalCloudletWrapper { +} \ No newline at end of file diff --git a/drivers/local/src/driver/cloudlet.rs b/drivers/local/src/driver/cloudlet.rs new file mode 100644 index 00000000..598a8449 --- /dev/null +++ b/drivers/local/src/driver/cloudlet.rs @@ -0,0 +1,36 @@ +use crate::exports::cloudlet::driver::bridge::{Address, Capabilities, GuestGenericCloudlet, RemoteController, Unit, UnitProposal}; + +use super::LocalCloudletWrapper; + +impl GuestGenericCloudlet for LocalCloudletWrapper { + fn new( + _cloud_identifier: String, + _name: String, + _id: Option, + _capabilities: Capabilities, + _controller: RemoteController, + ) -> Self { + Self {} + } + + /* This method expects that the Pterodactyl Allocations are only accessed by one atomic cloud instance */ + fn allocate_addresses(&self, _unit: UnitProposal) -> Result, String> { + Ok(Vec::new()) + } + + fn deallocate_addresses(&self, _addresses: Vec
) { + + } + + fn start_unit(&self, _unit: Unit) { + + } + + fn restart_unit(&self, _unit: Unit) { + + } + + fn stop_unit(&self, _unit: Unit) { + + } +} \ No newline at end of file diff --git a/drivers/local/src/log.rs b/drivers/local/src/log.rs new file mode 100644 index 00000000..ef513d3e --- /dev/null +++ b/drivers/local/src/log.rs @@ -0,0 +1,27 @@ +#[macro_export] +macro_rules! debug { + ($($arg:tt)*) => { + $crate::cloudlet::driver::log::log_string($crate::cloudlet::driver::log::Level::Debug, format!($($arg)*).as_str()); + }; +} + +#[macro_export] +macro_rules! info { + ($($arg:tt)*) => { + $crate::cloudlet::driver::log::log_string($crate::cloudlet::driver::log::Level::Info, format!($($arg)*).as_str()); + }; +} + +#[macro_export] +macro_rules! warn { + ($($arg:tt)*) => { + $crate::cloudlet::driver::log::log_string($crate::cloudlet::driver::log::Level::Warn, format!($($arg)*).as_str()); + }; +} + +#[macro_export] +macro_rules! error { + ($($arg:tt)*) => { + $crate::cloudlet::driver::log::log_string($crate::cloudlet::driver::log::Level::Error, format!($($arg)*).as_str()); + }; +} diff --git a/drivers/local/src/main.rs b/drivers/local/src/main.rs new file mode 100644 index 00000000..084b573f --- /dev/null +++ b/drivers/local/src/main.rs @@ -0,0 +1,24 @@ +#![no_main] + +use driver::{Local, LocalCloudletWrapper}; +use exports::cloudlet::driver::bridge::Guest; +use wit_bindgen::generate; + +mod config; +mod driver; +mod log; + +generate!({ + world: "driver", + path: "../../protocol/wit/", + additional_derives: [PartialEq, Eq], +}); + +struct Export; + +impl Guest for Export { + type GenericDriver = Local; + type GenericCloudlet = LocalCloudletWrapper; +} + +export!(Export); From ee2f89d29e1c4c3a87aa6144a5dae602cac68c7a Mon Sep 17 00:00:00 2001 From: HttpRafa <60099368+HttpRafa@users.noreply.github.com> Date: Sun, 12 Jan 2025 21:01:33 +0100 Subject: [PATCH 03/21] feat: Move storage thinks to storage modules --- .devcontainer/Dockerfile | 3 -- .github/docker/entrypoint.sh | 4 +- cli/src/application/profile.rs | 2 +- common/src/config.rs | 8 ++-- controller/src/application/auth.rs | 2 +- controller/src/application/cloudlet.rs | 4 +- controller/src/application/deployment.rs | 4 +- .../src/application/driver/wasm/process.rs | 2 +- controller/src/config.rs | 2 +- drivers/local/src/config.rs | 25 ----------- drivers/local/src/driver.rs | 32 +++++++++++--- drivers/local/src/driver/cloudlet.rs | 22 ++++------ drivers/local/src/main.rs | 2 +- drivers/local/src/storage.rs | 43 +++++++++++++++++++ drivers/pterodactyl/src/config.rs | 25 ----------- drivers/pterodactyl/src/driver/backend.rs | 9 ++-- drivers/pterodactyl/src/main.rs | 2 +- drivers/pterodactyl/src/storage.rs | 22 ++++++++++ 18 files changed, 118 insertions(+), 95 deletions(-) delete mode 100644 drivers/local/src/config.rs create mode 100644 drivers/local/src/storage.rs delete mode 100644 drivers/pterodactyl/src/config.rs create mode 100644 drivers/pterodactyl/src/storage.rs diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 40be7190..061cb545 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -26,9 +26,6 @@ USER ${REMOTE_USER} # Install Rustup and set default toolchains RUN rustup default nightly -# Install the wasm32-wasi target for nightly -RUN rustup target add wasm32-wasip1 --toolchain nightly - # Install binstall RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash diff --git a/.github/docker/entrypoint.sh b/.github/docker/entrypoint.sh index 339aba9d..d34888f2 100644 --- a/.github/docker/entrypoint.sh +++ b/.github/docker/entrypoint.sh @@ -25,9 +25,7 @@ if [ "$PTERODACTYL" = "true" ]; then echo "Download complete." fi -else - echo "PTERODACTYL is not set to true. Skipping download." fi # Run the main command -./controller "$@" \ No newline at end of file +exec ./controller "$@" \ No newline at end of file diff --git a/cli/src/application/profile.rs b/cli/src/application/profile.rs index 006af294..0b3eedb1 100644 --- a/cli/src/application/profile.rs +++ b/cli/src/application/profile.rs @@ -177,7 +177,7 @@ impl Profile { authorization: self.authorization.clone(), url: self.url.clone(), }; - stored_profile.save_to_file(&Storage::get_profile_file(&self.id)) + stored_profile.save_to_file(&Storage::get_profile_file(&self.id), true) } pub fn compute_id(name: &str) -> String { diff --git a/common/src/config.rs b/common/src/config.rs index 5f8fa0e4..689084ca 100644 --- a/common/src/config.rs +++ b/common/src/config.rs @@ -4,9 +4,11 @@ use anyhow::Result; use serde::{de::DeserializeOwned, Serialize}; pub trait SaveToTomlFile: Serialize { - fn save_to_file(&self, path: &Path) -> Result<()> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; + fn save_to_file(&self, path: &Path, create_parent: bool) -> Result<()> { + if create_parent { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } } fs::write(path, toml::to_string(self)?)?; Ok(()) diff --git a/controller/src/application/auth.rs b/controller/src/application/auth.rs index 77bb5ce3..90396542 100644 --- a/controller/src/application/auth.rs +++ b/controller/src/application/auth.rs @@ -176,7 +176,7 @@ impl Auth { token: token.to_string(), }; let user_path = Storage::get_user_file(username); - if stored_user.save_to_file(&user_path).is_err() { + if stored_user.save_to_file(&user_path, true).is_err() { error!( "Failed to save user to file: {}", &user_path.display() diff --git a/controller/src/application/cloudlet.rs b/controller/src/application/cloudlet.rs index 599dd408..26b1961e 100644 --- a/controller/src/application/cloudlet.rs +++ b/controller/src/application/cloudlet.rs @@ -215,7 +215,7 @@ impl Cloudlets { match self.add_cloudlet(cloudlet) { Ok(_) => { - stored_cloudlet.save_to_file(&Storage::get_cloudlet_file(name))?; + stored_cloudlet.save_to_file(&Storage::get_cloudlet_file(name), true)?; info!("Created cloudlet {}", name); Ok(CreationResult::Created) } @@ -393,7 +393,7 @@ impl Cloudlet { status: self.status.read().unwrap().clone(), controller: self.controller.clone(), }; - stored_cloudlet.save_to_file(&Storage::get_cloudlet_file(&self.name)) + stored_cloudlet.save_to_file(&Storage::get_cloudlet_file(&self.name), true) } } diff --git a/controller/src/application/deployment.rs b/controller/src/application/deployment.rs index f50fb415..c9023a6a 100644 --- a/controller/src/application/deployment.rs +++ b/controller/src/application/deployment.rs @@ -244,7 +244,7 @@ impl Deployments { ); self.add_deployment(deployment); - stored_deployment.save_to_file(&Storage::get_deployment_file(name))?; + stored_deployment.save_to_file(&Storage::get_deployment_file(name), true)?; info!("Created deployment {}", name); Ok(CreationResult::Created) } @@ -514,7 +514,7 @@ impl Deployment { resources: self.resources.clone(), spec: self.spec.clone(), }; - stored_deployment.save_to_file(&Storage::get_deployment_file(&self.name)) + stored_deployment.save_to_file(&Storage::get_deployment_file(&self.name), true) } } diff --git a/controller/src/application/driver/wasm/process.rs b/controller/src/application/driver/wasm/process.rs index 90f56e23..b34b114f 100644 --- a/controller/src/application/driver/wasm/process.rs +++ b/controller/src/application/driver/wasm/process.rs @@ -109,4 +109,4 @@ impl driver::process::Host for WasmDriverState { Err("Child process does not exist".to_string()) } } -} \ No newline at end of file +} diff --git a/controller/src/config.rs b/controller/src/config.rs index 2a560b2f..5153845a 100644 --- a/controller/src/config.rs +++ b/controller/src/config.rs @@ -94,7 +94,7 @@ impl Config { save = true; } if save { - if let Err(error) = config.save_to_file(&Storage::get_primary_config_file()) { + if let Err(error) = config.save_to_file(&Storage::get_primary_config_file(), true) { error!( "Failed to save generated configuration to file: {}", &error diff --git a/drivers/local/src/config.rs b/drivers/local/src/config.rs deleted file mode 100644 index 8984b45d..00000000 --- a/drivers/local/src/config.rs +++ /dev/null @@ -1,25 +0,0 @@ -use anyhow::Result; -use serde::{de::DeserializeOwned, Serialize}; -use std::{fs, path::Path}; - -pub const CONFIG_DIRECTORY: &str = "/configs"; - -pub trait SaveToTomlFile: Serialize { - fn save_to_file(&self, path: &Path, create_parent: bool) -> Result<()> { - if let Some(parent) = path.parent() { - if create_parent { - fs::create_dir_all(parent)?; - } - } - fs::write(path, toml::to_string(self)?)?; - Ok(()) - } -} - -pub trait LoadFromTomlFile: DeserializeOwned { - fn load_from_file(path: &Path) -> Result { - let data = fs::read_to_string(path)?; - let config = toml::from_str(&data)?; - Ok(config) - } -} diff --git a/drivers/local/src/driver.rs b/drivers/local/src/driver.rs index 86c08811..7e0f9b3e 100644 --- a/drivers/local/src/driver.rs +++ b/drivers/local/src/driver.rs @@ -1,4 +1,12 @@ -use crate::exports::cloudlet::driver::bridge::{Capabilities, GenericCloudlet, GuestGenericDriver, Information, RemoteController}; +use std::fs; + +use crate::{ + error, + exports::cloudlet::driver::bridge::{ + Capabilities, GenericCloudlet, GuestGenericDriver, Information, RemoteController, + }, + storage::Storage, +}; pub mod cloudlet; @@ -14,16 +22,27 @@ pub struct Local { impl GuestGenericDriver for Local { fn new(cloud_identifier: String) -> Self { - Self { - cloud_identifier, - } + Self { cloud_identifier } } fn init(&self) -> Information { + let mut ready = true; + + let tmp_dir = Storage::get_temporary_folder(); + if tmp_dir.exists() { + if let Err(error) = fs::remove_dir_all(tmp_dir) { + error!( + "Failed to remove temporary directory: {}", + error + ); + ready = false; + } + } + Information { authors: AUTHORS.iter().map(|&author| author.to_string()).collect(), version: VERSION.to_string(), - ready: false, + ready, } } @@ -37,5 +56,4 @@ impl GuestGenericDriver for Local { } } -pub struct LocalCloudletWrapper { -} \ No newline at end of file +pub struct LocalCloudletWrapper {} diff --git a/drivers/local/src/driver/cloudlet.rs b/drivers/local/src/driver/cloudlet.rs index 598a8449..ec77cb47 100644 --- a/drivers/local/src/driver/cloudlet.rs +++ b/drivers/local/src/driver/cloudlet.rs @@ -1,4 +1,6 @@ -use crate::exports::cloudlet::driver::bridge::{Address, Capabilities, GuestGenericCloudlet, RemoteController, Unit, UnitProposal}; +use crate::exports::cloudlet::driver::bridge::{ + Address, Capabilities, GuestGenericCloudlet, RemoteController, Unit, UnitProposal, +}; use super::LocalCloudletWrapper; @@ -18,19 +20,11 @@ impl GuestGenericCloudlet for LocalCloudletWrapper { Ok(Vec::new()) } - fn deallocate_addresses(&self, _addresses: Vec
) { + fn deallocate_addresses(&self, _addresses: Vec
) {} - } - - fn start_unit(&self, _unit: Unit) { - - } + fn start_unit(&self, _unit: Unit) {} - fn restart_unit(&self, _unit: Unit) { + fn restart_unit(&self, _unit: Unit) {} - } - - fn stop_unit(&self, _unit: Unit) { - - } -} \ No newline at end of file + fn stop_unit(&self, _unit: Unit) {} +} diff --git a/drivers/local/src/main.rs b/drivers/local/src/main.rs index 084b573f..c627f16c 100644 --- a/drivers/local/src/main.rs +++ b/drivers/local/src/main.rs @@ -4,9 +4,9 @@ use driver::{Local, LocalCloudletWrapper}; use exports::cloudlet::driver::bridge::Guest; use wit_bindgen::generate; -mod config; mod driver; mod log; +mod storage; generate!({ world: "driver", diff --git a/drivers/local/src/storage.rs b/drivers/local/src/storage.rs new file mode 100644 index 00000000..ac8510f7 --- /dev/null +++ b/drivers/local/src/storage.rs @@ -0,0 +1,43 @@ +/* +All the storage related functions are implemented here. +This makes it easier to change them in the future +*/ + +use std::path::PathBuf; + +/* Configs */ +//const CONFIG_DIRECTORY: &str = "/configs"; + +/* Data */ +const DATA_DIRECTORY: &str = "/data"; +//const TEMPLATES_DIRECTORY: &str = "templates"; +const UNITS_DIRECTORY: &str = "units"; + +/* Units */ +const TEMPORARY_DIRECTORY: &str = "temporary"; +//const PERMANENT_DIRECTORY: &str = "permanent"; + +pub struct Storage; + +impl Storage { + /* Configs */ + /*pub fn get_configs_folder() -> PathBuf { + PathBuf::from(CONFIG_DIRECTORY) + }*/ + + /* Data */ + pub fn get_data_folder() -> PathBuf { + PathBuf::from(DATA_DIRECTORY) + } + pub fn get_units_folder() -> PathBuf { + Self::get_data_folder().join(UNITS_DIRECTORY) + } + + /* Units */ + pub fn get_temporary_folder() -> PathBuf { + Self::get_units_folder().join(TEMPORARY_DIRECTORY) + } + /*pub fn get_permanent_folder() -> PathBuf { + Self::get_units_folder().join(PERMANENT_DIRECTORY) + }*/ +} diff --git a/drivers/pterodactyl/src/config.rs b/drivers/pterodactyl/src/config.rs deleted file mode 100644 index 8984b45d..00000000 --- a/drivers/pterodactyl/src/config.rs +++ /dev/null @@ -1,25 +0,0 @@ -use anyhow::Result; -use serde::{de::DeserializeOwned, Serialize}; -use std::{fs, path::Path}; - -pub const CONFIG_DIRECTORY: &str = "/configs"; - -pub trait SaveToTomlFile: Serialize { - fn save_to_file(&self, path: &Path, create_parent: bool) -> Result<()> { - if let Some(parent) = path.parent() { - if create_parent { - fs::create_dir_all(parent)?; - } - } - fs::write(path, toml::to_string(self)?)?; - Ok(()) - } -} - -pub trait LoadFromTomlFile: DeserializeOwned { - fn load_from_file(path: &Path) -> Result { - let data = fs::read_to_string(path)?; - let config = toml::from_str(&data)?; - Ok(config) - } -} diff --git a/drivers/pterodactyl/src/driver/backend.rs b/drivers/pterodactyl/src/driver/backend.rs index 9f99dbbb..825ab6dc 100644 --- a/drivers/pterodactyl/src/driver/backend.rs +++ b/drivers/pterodactyl/src/driver/backend.rs @@ -1,5 +1,6 @@ -use std::{collections::HashMap, path::Path}; +use std::collections::HashMap; +use ::common::config::{LoadFromTomlFile, SaveToTomlFile}; use allocation::{BAllocation, BCAllocation}; use anyhow::Result; use common::{BBody, BList, BObject}; @@ -14,9 +15,9 @@ use user::BUser; use crate::{ cloudlet::driver::http::{send_http_request, Header, Method, Response}, - config::{LoadFromTomlFile, SaveToTomlFile, CONFIG_DIRECTORY}, debug, error, exports::cloudlet::driver::bridge::Unit, + storage::Storage, warn, }; @@ -31,8 +32,6 @@ mod node; pub mod server; mod user; -const BACKEND_FILE: &str = "backend.toml"; - /* Endpoints */ const APPLICATION_ENDPOINT: &str = "api/application"; const CLIENT_ENDPOINT: &str = "api/client"; @@ -101,7 +100,7 @@ impl Backend { } fn load_or_empty() -> Self { - let path = Path::new(CONFIG_DIRECTORY).join(BACKEND_FILE); + let path = Storage::get_backend_config_file(); if path.exists() { Self::load_from_file(&path).unwrap_or_else(|err| { warn!("Failed to read backend configuration from file: {}", err); diff --git a/drivers/pterodactyl/src/main.rs b/drivers/pterodactyl/src/main.rs index 724c63e5..c372188a 100644 --- a/drivers/pterodactyl/src/main.rs +++ b/drivers/pterodactyl/src/main.rs @@ -4,9 +4,9 @@ use driver::{Pterodactyl, PterodactylCloudletWrapper}; use exports::cloudlet::driver::bridge::Guest; use wit_bindgen::generate; -mod config; mod driver; mod log; +mod storage; generate!({ world: "driver", diff --git a/drivers/pterodactyl/src/storage.rs b/drivers/pterodactyl/src/storage.rs new file mode 100644 index 00000000..f8a43318 --- /dev/null +++ b/drivers/pterodactyl/src/storage.rs @@ -0,0 +1,22 @@ +/* +All the storage related functions are implemented here. +This makes it easier to change them in the future +*/ + +use std::path::PathBuf; + +/* Configs */ +const CONFIG_DIRECTORY: &str = "/configs"; +const BACKEND_CONFIG_FILE: &str = "backend.toml"; + +pub struct Storage; + +impl Storage { + /* Configs */ + pub fn get_configs_folder() -> PathBuf { + PathBuf::from(CONFIG_DIRECTORY) + } + pub fn get_backend_config_file() -> PathBuf { + Storage::get_configs_folder().join(BACKEND_CONFIG_FILE) + } +} From e9d622179f5b34cb2ec01501994675b230620a37 Mon Sep 17 00:00:00 2001 From: HttpRafa <60099368+HttpRafa@users.noreply.github.com> Date: Wed, 15 Jan 2025 13:23:31 +0100 Subject: [PATCH 04/21] feat: Work on the local drivers template system --- cli/src/application/profile.rs | 2 +- controller/src/application/driver/wasm.rs | 1 + .../src/application/driver/wasm/config.rs | 10 +- .../src/application/driver/wasm/platform.rs | 14 ++ .../src/application/driver/wasm/process.rs | 62 ++++- drivers/local/Cargo.toml | 3 + drivers/local/src/driver.rs | 58 ++++- drivers/local/src/driver/cloudlet.rs | 15 +- drivers/local/src/driver/template.rs | 230 ++++++++++++++++++ drivers/local/src/storage.rs | 19 ++ drivers/local/templates/paper/template.toml | 19 ++ protocol/wit/driver.wit | 24 +- 12 files changed, 439 insertions(+), 18 deletions(-) create mode 100644 controller/src/application/driver/wasm/platform.rs create mode 100644 drivers/local/src/driver/template.rs create mode 100644 drivers/local/templates/paper/template.toml diff --git a/cli/src/application/profile.rs b/cli/src/application/profile.rs index 0b3eedb1..8de8ffb3 100644 --- a/cli/src/application/profile.rs +++ b/cli/src/application/profile.rs @@ -86,7 +86,7 @@ impl Profiles { profiles.add_profile(profile); } - progress.success(format!("Loaded {} profiles(s)", profiles.profiles.len())); + progress.success(format!("Loaded {} profile(s)", profiles.profiles.len())); progress.end(); profiles } diff --git a/controller/src/application/driver/wasm.rs b/controller/src/application/driver/wasm.rs index 1155d0fb..3204c73c 100644 --- a/controller/src/application/driver/wasm.rs +++ b/controller/src/application/driver/wasm.rs @@ -23,6 +23,7 @@ mod config; mod cloudlet; mod http; mod log; +mod platform; mod process; pub mod generated { diff --git a/controller/src/application/driver/wasm/config.rs b/controller/src/application/driver/wasm/config.rs index 8fb86c8a..181ce6a9 100644 --- a/controller/src/application/driver/wasm/config.rs +++ b/controller/src/application/driver/wasm/config.rs @@ -1,6 +1,7 @@ use common::config::{LoadFromTomlFile, SaveToTomlFile}; use regex::Regex; use serde::{Deserialize, Serialize}; +use simplelog::warn; #[derive(Serialize, Deserialize, Default)] pub struct WasmConfig { @@ -12,10 +13,15 @@ impl SaveToTomlFile for WasmConfig {} impl WasmConfig { pub fn get_config(&self, name: &str) -> Option<&DriverConfig> { - let regex = Regex::new(name).ok()?; self.drivers .iter() - .find(|driver| regex.is_match(&driver.name)) + .find(|driver| match Regex::new(&driver.name) { + Ok(regex) => regex.is_match(name), + Err(error) => { + warn!("Failed to compile driver name regex: {}", error); + false + } + }) } } diff --git a/controller/src/application/driver/wasm/platform.rs b/controller/src/application/driver/wasm/platform.rs new file mode 100644 index 00000000..0b328f31 --- /dev/null +++ b/controller/src/application/driver/wasm/platform.rs @@ -0,0 +1,14 @@ +use super::{ + generated::cloudlet::driver::{self, platform::Os}, + WasmDriverState, +}; + +impl driver::platform::Host for WasmDriverState { + fn get_os(&mut self) -> Os { + if cfg!(target_os = "windows") { + Os::Windows + } else { + Os::Unix + } + } +} diff --git a/controller/src/application/driver/wasm/process.rs b/controller/src/application/driver/wasm/process.rs index b34b114f..3ba4e897 100644 --- a/controller/src/application/driver/wasm/process.rs +++ b/controller/src/application/driver/wasm/process.rs @@ -1,14 +1,43 @@ use std::{ io::{Read, Write}, - process::Command, + path::PathBuf, + process::{Command, Stdio}, }; -use super::{generated::cloudlet::driver, WasmDriverState}; +use crate::storage::Storage; + +use super::{ + generated::cloudlet::driver::{ + self, + process::{Directory, Reference}, + }, + WasmDriverState, +}; impl driver::process::Host for WasmDriverState { - fn spawn_process(&mut self, command: String, args: Vec) -> Result { + fn spawn_process( + &mut self, + command: String, + args: Vec, + directory: Directory, + ) -> Result { let driver = self.handle.upgrade().ok_or("Failed to upgrade handle")?; - let command = Command::new(command).args(args).spawn(); + let process_dir = match &directory.reference { + Reference::Controller => PathBuf::from(".").join(&directory.path), + Reference::Data => { + Storage::get_data_folder_for_driver(&driver.name).join(&directory.path) + } + Reference::Configs => { + Storage::get_config_folder_for_driver(&driver.name).join(&directory.path) + } + }; + let command = Command::new(command) + .args(args) + .current_dir(process_dir) + .stdin(Stdio::inherit()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn(); match command { Ok(child) => { let pid = child.id(); @@ -42,6 +71,31 @@ impl driver::process::Host for WasmDriverState { } } + fn try_wait_process(&mut self, pid: u32) -> Result, String> { + let driver = self.handle.upgrade().ok_or("Failed to upgrade handle")?; + let mut child_processes = driver + .data + .child_processes + .write() + .map_err(|_| "Failed to acquire write lock on child processes")?; + + if let Some(child) = child_processes.get_mut(&pid) { + child + .try_wait() + .map_err(|error| format!("Failed to wait for child process: {}", error)) + .map(|status| { + if let Some(status) = status { + child_processes.remove(&pid); + Some(status.code().unwrap_or(0)) + } else { + None + } + }) + } else { + Ok(None) + } + } + fn read_stdout(&mut self, pid: u32) -> Result, String> { let driver = self.handle.upgrade().ok_or("Failed to upgrade handle")?; let mut child_processes = driver diff --git a/drivers/local/Cargo.toml b/drivers/local/Cargo.toml index 6beb1fe1..a64abab4 100644 --- a/drivers/local/Cargo.toml +++ b/drivers/local/Cargo.toml @@ -16,6 +16,9 @@ wit-bindgen = "0.37.0" # Error handling anyhow = "1.0.95" +# Regex parsing +regex = "1.11.1" + # Configuration serde = { version = "1.0.217", features = ["derive"] } toml = "0.8.19" diff --git a/drivers/local/src/driver.rs b/drivers/local/src/driver.rs index 7e0f9b3e..f0089cc5 100644 --- a/drivers/local/src/driver.rs +++ b/drivers/local/src/driver.rs @@ -1,14 +1,20 @@ -use std::fs; +use std::{fs, rc::Rc, sync::RwLock}; + +use cloudlet::LocalCloudlet; +use template::Templates; use crate::{ error, exports::cloudlet::driver::bridge::{ - Capabilities, GenericCloudlet, GuestGenericDriver, Information, RemoteController, + Capabilities, GenericCloudlet, GuestGenericCloudlet, GuestGenericDriver, Information, + RemoteController, }, + info, storage::Storage, }; pub mod cloudlet; +mod template; // Include the build information generated by build.rs include!(concat!(env!("OUT_DIR"), "/build_info.rs")); @@ -18,11 +24,21 @@ pub const AUTHORS: [&str; 1] = ["HttpRafa"]; pub struct Local { /* Cloud Identification */ cloud_identifier: String, + + /* Templates */ + templates: RwLock, + + /* Cloudlets that this driver handles */ + cloudlets: RwLock>>, } impl GuestGenericDriver for Local { fn new(cloud_identifier: String) -> Self { - Self { cloud_identifier } + Self { + cloud_identifier, + templates: RwLock::new(Templates::new()), + cloudlets: RwLock::new(Vec::new()), + } } fn init(&self) -> Information { @@ -39,6 +55,16 @@ impl GuestGenericDriver for Local { } } + // Load all templates + { + let mut templates = self + .templates + .write() + .expect("Failed to get lock on templates"); + templates.load_all(); + templates.prepare_all(); + } + Information { authors: AUTHORS.iter().map(|&author| author.to_string()).collect(), version: VERSION.to_string(), @@ -48,12 +74,28 @@ impl GuestGenericDriver for Local { fn init_cloudlet( &self, - _name: String, - _capabilities: Capabilities, - _controller: RemoteController, + name: String, + capabilities: Capabilities, + controller: RemoteController, ) -> Result { - Err("Not implemented".to_string()) + let wrapper = LocalCloudletWrapper::new( + self.cloud_identifier.clone(), + name.clone(), + None, + capabilities, + controller, + ); + // Add cloudlet to cloudlets list + let mut cloudlets = self + .cloudlets + .write() + .expect("Failed to get lock on cloudlets"); + cloudlets.push(wrapper.inner.clone()); + info!("Cloudlet {} was added", name); + Ok(GenericCloudlet::new(wrapper)) } } -pub struct LocalCloudletWrapper {} +pub struct LocalCloudletWrapper { + pub inner: Rc, +} diff --git a/drivers/local/src/driver/cloudlet.rs b/drivers/local/src/driver/cloudlet.rs index ec77cb47..5c71046e 100644 --- a/drivers/local/src/driver/cloudlet.rs +++ b/drivers/local/src/driver/cloudlet.rs @@ -1,3 +1,5 @@ +use std::rc::Rc; + use crate::exports::cloudlet::driver::bridge::{ Address, Capabilities, GuestGenericCloudlet, RemoteController, Unit, UnitProposal, }; @@ -10,9 +12,13 @@ impl GuestGenericCloudlet for LocalCloudletWrapper { _name: String, _id: Option, _capabilities: Capabilities, - _controller: RemoteController, + controller: RemoteController, ) -> Self { - Self {} + Self { + inner: Rc::new(LocalCloudlet { + _controller: controller, + }), + } } /* This method expects that the Pterodactyl Allocations are only accessed by one atomic cloud instance */ @@ -28,3 +34,8 @@ impl GuestGenericCloudlet for LocalCloudletWrapper { fn stop_unit(&self, _unit: Unit) {} } + +pub struct LocalCloudlet { + /* Informations about the cloudlet */ + pub _controller: RemoteController, +} diff --git a/drivers/local/src/driver/template.rs b/drivers/local/src/driver/template.rs new file mode 100644 index 00000000..0286f9fd --- /dev/null +++ b/drivers/local/src/driver/template.rs @@ -0,0 +1,230 @@ +use std::fs; + +use common::config::LoadFromTomlFile; +use serde::{Deserialize, Serialize}; +use stored::StoredTemplate; + +use crate::{ + cloudlet::driver::{ + platform::{get_os, Os}, + process::{read_stdout, spawn_process, try_wait_process, Directory, Reference}, + }, + info, + storage::Storage, + warn, +}; + +pub struct Templates { + /* Templates */ + pub templates: Vec