diff --git a/Cargo.lock b/Cargo.lock index 34d3b71b..f4c502e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6355,6 +6355,7 @@ dependencies = [ "urlencoding", "uuid", "wiremock", + "zstd", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 94540432..1fa9ca93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,6 +77,7 @@ dialoguer = { version = "0.11", features = ["fuzzy-select"] } indicatif = "0.17" brotli = "3.4.0" serde_path_to_error = "0.1.14" +zstd = "0.13" deadpool-lapin = "0.12.1" docker-compose-types = "0.7.0" actix-casbin-auth = { git = "https://github.com/casbin-rs/actix-casbin-auth.git"} diff --git a/docs/STACKER_YML_REFERENCE.md b/docs/STACKER_YML_REFERENCE.md index e2f429b7..0cc1fe77 100644 --- a/docs/STACKER_YML_REFERENCE.md +++ b/docs/STACKER_YML_REFERENCE.md @@ -606,7 +606,7 @@ deploy: registry: username: "${DOCKER_USERNAME}" password: "${DOCKER_PASSWORD}" - # server: "https://index.docker.io/v1/" # Docker Hub (default) + # server: "docker.io" # Docker Hub (default) ``` > **Security tip:** Use environment variables or `${VAR}` syntax to keep credentials out of version control. diff --git a/src/bin/stacker.rs b/src/bin/stacker.rs index d9b87c47..5a6162d7 100644 --- a/src/bin/stacker.rs +++ b/src/bin/stacker.rs @@ -96,6 +96,9 @@ enum StackerCommands { /// Deployment target: local, cloud, server #[arg(long, value_name = "TARGET")] target: Option, + /// Deploy environment/profile, e.g. development, staging, production + #[arg(long = "env", alias = "environment", value_name = "ENVIRONMENT")] + environment: Option, /// Path to stacker.yml (default: ./stacker.yml) #[arg(long, value_name = "FILE")] file: Option, @@ -1086,6 +1089,7 @@ fn get_command( } StackerCommands::Deploy { target, + environment, file, dry_run, force_rebuild, @@ -1105,6 +1109,7 @@ fn get_command( dry_run, force_rebuild, ) + .with_environment(environment) .with_remote_overrides(project, key, server) .with_key_id(key_id) .with_watch(watch, no_watch) @@ -1563,6 +1568,51 @@ fn get_command( mod tests { use super::*; + #[test] + fn test_deploy_parses_environment_alias() { + let cli = Cli::try_parse_from([ + "stacker", + "deploy", + "--target", + "cloud", + "--env", + "production", + ]) + .unwrap(); + + match cli.command.unwrap() { + StackerCommands::Deploy { + target, + environment, + .. + } => { + assert_eq!(target.as_deref(), Some("cloud")); + assert_eq!(environment.as_deref(), Some("production")); + } + _ => panic!("expected deploy command"), + } + } + + #[test] + fn test_deploy_parses_environment_long_alias() { + let cli = Cli::try_parse_from([ + "stacker", + "deploy", + "--target", + "cloud", + "--environment", + "staging", + ]) + .unwrap(); + + match cli.command.unwrap() { + StackerCommands::Deploy { environment, .. } => { + assert_eq!(environment.as_deref(), Some("staging")); + } + _ => panic!("expected deploy command"), + } + } + #[test] fn test_pipe_scan_parses_without_selector() { let cli = Cli::try_parse_from(["stacker", "pipe", "scan"]).unwrap(); diff --git a/src/cli/config_bundle.rs b/src/cli/config_bundle.rs new file mode 100644 index 00000000..26dc406e --- /dev/null +++ b/src/cli/config_bundle.rs @@ -0,0 +1,616 @@ +use std::collections::BTreeMap; +use std::fs::File; +use std::path::{Component, Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use serde_json::json; +use sha2::{Digest, Sha256}; +use tar::{Builder, Header}; +use zstd::stream::write::Encoder; + +use crate::cli::error::CliError; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ConfigBundleFile { + pub source_path: String, + pub destination_path: String, + pub mode: String, + pub size: u64, + pub sha256: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ConfigBundleManifest { + pub version: u32, + pub environment: String, + pub files: Vec, +} + +#[derive(Debug, Clone)] +pub struct ConfigBundleArtifacts { + pub environment: String, + pub manifest_path: PathBuf, + pub archive_path: PathBuf, + pub remote_compose_path: PathBuf, + pub manifest: ConfigBundleManifest, + pub config_files: Vec, +} + +impl ConfigBundleArtifacts { + pub fn artifact_metadata(&self) -> serde_json::Value { + let files: Vec = self + .manifest + .files + .iter() + .map(|file| { + json!({ + "source_path": file.source_path, + "destination_path": file.destination_path, + "mode": file.mode, + "size": file.size, + "sha256": file.sha256, + "content_hidden": is_secret_like_path(&file.source_path), + }) + }) + .collect(); + + json!({ + "environment": self.environment, + "manifest_path": self.manifest_path.to_string_lossy(), + "archive_path": self.archive_path.to_string_lossy(), + "remote_compose_path": self.remote_compose_path.to_string_lossy(), + "config_files": files, + }) + } +} + +pub fn build_config_bundle( + project_dir: &Path, + environment: &str, + compose_path: &Path, + env_file: Option<&Path>, +) -> Result { + validate_environment_name(environment)?; + + let project_root = project_dir.canonicalize()?; + let compose_canonical = compose_path.canonicalize()?; + ensure_inside_project(&project_root, &compose_canonical)?; + let compose_dir = compose_canonical + .parent() + .ok_or_else(|| validation_error("compose file must have a parent directory"))?; + + let output_dir = project_root.join(".stacker/deploy").join(environment); + std::fs::create_dir_all(&output_dir)?; + let manifest_path = output_dir.join("config-bundle.manifest.json"); + let archive_path = output_dir.join("config-bundle.tar.zst"); + let remote_compose_path = output_dir.join("docker-compose.remote.yml"); + + let compose_content = std::fs::read_to_string(&compose_canonical)?; + let mut compose_yaml: serde_yaml::Value = serde_yaml::from_str(&compose_content)?; + let mut collected = BTreeMap::::new(); + + if let Some(env_file) = env_file { + let resolved = resolve_reference_path(&project_root, &project_root, env_file)?; + collect_file(&project_root, environment, resolved, &mut collected)?; + } + + rewrite_compose_references( + &project_root, + compose_dir, + environment, + &mut compose_yaml, + &mut collected, + )?; + + let rewritten_compose = serde_yaml::to_string(&compose_yaml) + .map_err(|err| validation_error(format!("failed to write remote compose: {err}")))?; + std::fs::write(&remote_compose_path, &rewritten_compose)?; + + let mut files: Vec = collected + .values() + .map(|file| ConfigBundleFile { + source_path: file.source_path.clone(), + destination_path: file.destination_path.clone(), + mode: file.mode.clone(), + size: file.bytes.len() as u64, + sha256: sha256_hex(&file.bytes), + }) + .collect(); + files.sort_by(|left, right| left.source_path.cmp(&right.source_path)); + + let manifest = ConfigBundleManifest { + version: 1, + environment: environment.to_string(), + files, + }; + let manifest_json = serde_json::to_string_pretty(&manifest) + .map_err(|err| validation_error(format!("failed to serialize manifest: {err}")))?; + std::fs::write(&manifest_path, manifest_json)?; + write_archive(&archive_path, collected.values())?; + + let mut config_files = Vec::new(); + config_files.push(json!({ + "name": "docker-compose.yml", + "content": rewritten_compose, + "content_type": "application/x-yaml", + "destination_path": "docker-compose.yml", + "file_mode": "0644", + "owner": "root", + "group": "root" + })); + + for file in collected.values() { + let content = String::from_utf8(file.bytes.clone()).map_err(|_| { + validation_error(format!( + "config file '{}' must be UTF-8 text to upload in the deploy payload", + file.source_path + )) + })?; + config_files.push(json!({ + "name": Path::new(&file.source_path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(file.source_path.as_str()), + "content": content, + "content_type": "text/plain", + "destination_path": file.destination_path, + "file_mode": file.mode, + "owner": "root", + "group": "root" + })); + } + + Ok(ConfigBundleArtifacts { + environment: environment.to_string(), + manifest_path, + archive_path, + remote_compose_path, + manifest, + config_files, + }) +} + +#[derive(Debug, Clone)] +struct CollectedFile { + source_path: String, + destination_path: String, + mode: String, + bytes: Vec, +} + +fn rewrite_compose_references( + project_root: &Path, + compose_dir: &Path, + environment: &str, + compose_yaml: &mut serde_yaml::Value, + collected: &mut BTreeMap, +) -> Result<(), CliError> { + let Some(services) = mapping_mut(compose_yaml) + .and_then(|root| root.get_mut(serde_yaml::Value::String("services".to_string()))) + .and_then(mapping_mut) + else { + return Ok(()); + }; + + for service in services.values_mut() { + let Some(service_map) = mapping_mut(service) else { + continue; + }; + + if let Some(env_file_value) = + service_map.get_mut(serde_yaml::Value::String("env_file".to_string())) + { + rewrite_env_file( + project_root, + compose_dir, + environment, + env_file_value, + collected, + )?; + } + + if let Some(volumes_value) = + service_map.get_mut(serde_yaml::Value::String("volumes".to_string())) + { + rewrite_volumes( + project_root, + compose_dir, + environment, + volumes_value, + collected, + )?; + } + } + + Ok(()) +} + +fn rewrite_env_file( + project_root: &Path, + compose_dir: &Path, + environment: &str, + value: &mut serde_yaml::Value, + collected: &mut BTreeMap, +) -> Result<(), CliError> { + match value { + serde_yaml::Value::String(path) => { + let remote = + collect_reference(project_root, compose_dir, environment, path, collected)?; + *path = remote; + } + serde_yaml::Value::Sequence(values) => { + for item in values { + if let serde_yaml::Value::String(path) = item { + let remote = + collect_reference(project_root, compose_dir, environment, path, collected)?; + *path = remote; + } + } + } + _ => {} + } + + Ok(()) +} + +fn rewrite_volumes( + project_root: &Path, + compose_dir: &Path, + environment: &str, + value: &mut serde_yaml::Value, + collected: &mut BTreeMap, +) -> Result<(), CliError> { + let serde_yaml::Value::Sequence(volumes) = value else { + return Ok(()); + }; + + for volume in volumes { + let serde_yaml::Value::String(volume_spec) = volume else { + continue; + }; + let Some((source, rest)) = parse_bind_mount(volume_spec) else { + continue; + }; + let remote = collect_reference(project_root, compose_dir, environment, source, collected)?; + *volume_spec = format!("{remote}:{rest}"); + } + + Ok(()) +} + +fn parse_bind_mount(volume_spec: &str) -> Option<(&str, &str)> { + let (source, rest) = volume_spec.split_once(':')?; + if source.starts_with('.') + || source.starts_with('/') + || source.starts_with('~') + || source.contains(std::path::MAIN_SEPARATOR) + { + Some((source, rest)) + } else { + None + } +} + +fn collect_reference( + project_root: &Path, + base_dir: &Path, + environment: &str, + reference: &str, + collected: &mut BTreeMap, +) -> Result { + let resolved = resolve_reference_path(project_root, base_dir, Path::new(reference))?; + let collected_file = collect_file(project_root, environment, resolved, collected)?; + Ok(collected_file.destination_path.clone()) +} + +fn collect_file<'a>( + project_root: &Path, + environment: &str, + path: PathBuf, + collected: &'a mut BTreeMap, +) -> Result<&'a CollectedFile, CliError> { + let canonical = path.canonicalize()?; + ensure_inside_project(project_root, &canonical)?; + + if canonical.is_dir() { + return Err(validation_error(format!( + "directory mounts are not supported in config bundles: {}", + display_project_path(project_root, &canonical) + ))); + } + if !canonical.is_file() { + return Err(validation_error(format!( + "config bundle path is not a file: {}", + canonical.display() + ))); + } + + if !collected.contains_key(&canonical) { + let source_path = display_project_path(project_root, &canonical); + let destination_path = format!( + "/opt/stacker/deployments/{environment}/files/{}", + source_path.replace('\\', "/") + ); + collected.insert( + canonical.clone(), + CollectedFile { + source_path, + destination_path, + mode: "0644".to_string(), + bytes: std::fs::read(&canonical)?, + }, + ); + } + + Ok(collected + .get(&canonical) + .expect("collected file was inserted")) +} + +fn write_archive<'a>( + archive_path: &Path, + files: impl IntoIterator, +) -> Result<(), CliError> { + let archive_file = File::create(archive_path)?; + let encoder = Encoder::new(archive_file, 0) + .map_err(|err| validation_error(format!("failed to create zstd archive: {err}")))?; + let mut tar = Builder::new(encoder); + + for file in files { + let mut header = Header::new_gnu(); + header.set_size(file.bytes.len() as u64); + header.set_mode(0o644); + header.set_mtime(0); + header.set_cksum(); + tar.append_data(&mut header, &file.source_path, file.bytes.as_slice())?; + } + + let encoder = tar.into_inner()?; + encoder + .finish() + .map_err(|err| validation_error(format!("failed to finish zstd archive: {err}")))?; + Ok(()) +} + +fn resolve_reference_path( + project_root: &Path, + base_dir: &Path, + reference: &Path, +) -> Result { + if reference.is_absolute() { + return Ok(reference.to_path_buf()); + } + + if reference.starts_with("~") { + return Err(validation_error(format!( + "home-relative config paths are not supported: {}", + reference.display() + ))); + } + + let base = if reference + .components() + .any(|component| matches!(component, Component::ParentDir)) + { + base_dir.join(reference).canonicalize()? + } else { + base_dir.join(reference) + }; + + let canonical = base.canonicalize()?; + ensure_inside_project(project_root, &canonical)?; + Ok(canonical) +} + +fn ensure_inside_project(project_root: &Path, path: &Path) -> Result<(), CliError> { + if path.starts_with(project_root) { + return Ok(()); + } + + Err(validation_error(format!( + "config bundle path must stay inside the project directory: {}", + path.display() + ))) +} + +fn display_project_path(project_root: &Path, path: &Path) -> String { + path.strip_prefix(project_root) + .unwrap_or(path) + .to_string_lossy() + .replace('\\', "/") +} + +fn validate_environment_name(environment: &str) -> Result<(), CliError> { + if !environment.is_empty() + && environment + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_')) + { + return Ok(()); + } + + Err(validation_error(format!( + "environment name must contain only letters, digits, '-' or '_': {environment}" + ))) +} + +fn sha256_hex(bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + format!("{:x}", hasher.finalize()) +} + +fn is_secret_like_path(path: &str) -> bool { + let file_name = Path::new(path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(path) + .to_ascii_lowercase(); + + file_name == ".env" + || file_name.ends_with(".env") + || file_name.contains("secret") + || file_name.contains("password") + || file_name.contains("private") + || file_name.ends_with(".key") +} + +fn mapping_mut(value: &mut serde_yaml::Value) -> Option<&mut serde_yaml::Mapping> { + match value { + serde_yaml::Value::Mapping(mapping) => Some(mapping), + _ => None, + } +} + +fn validation_error(message: impl Into) -> CliError { + CliError::ConfigValidation(message.into()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn build_config_bundle_collects_env_file_and_file_mounts_for_environment() { + let dir = TempDir::new().unwrap(); + let compose_dir = dir.path().join("docker/production"); + std::fs::create_dir_all(&compose_dir).unwrap(); + std::fs::write(compose_dir.join(".env"), "RUST_LOG=warning\n").unwrap(); + std::fs::write(compose_dir.join("nginx.conf"), "events {}\n").unwrap(); + std::fs::write( + compose_dir.join("compose.yml"), + r#" +services: + api: + image: device-api:latest + env_file: + - .env + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro +"#, + ) + .unwrap(); + + let artifacts = build_config_bundle( + dir.path(), + "production", + &compose_dir.join("compose.yml"), + None, + ) + .expect("bundle should be built"); + + assert_eq!(artifacts.environment, "production"); + assert!(artifacts + .manifest_path + .ends_with(".stacker/deploy/production/config-bundle.manifest.json")); + assert!(artifacts + .archive_path + .ends_with(".stacker/deploy/production/config-bundle.tar.zst")); + assert!(artifacts + .remote_compose_path + .ends_with(".stacker/deploy/production/docker-compose.remote.yml")); + assert!(artifacts.manifest_path.exists()); + assert!(artifacts.archive_path.exists()); + assert!(artifacts.remote_compose_path.exists()); + + let sources: Vec<&str> = artifacts + .manifest + .files + .iter() + .map(|file| file.source_path.as_str()) + .collect(); + assert!(sources.contains(&"docker/production/.env")); + assert!(sources.contains(&"docker/production/nginx.conf")); + + let remote_compose = std::fs::read_to_string(&artifacts.remote_compose_path).unwrap(); + assert!(remote_compose + .contains("/opt/stacker/deployments/production/files/docker/production/.env")); + assert!(remote_compose.contains("/opt/stacker/deployments/production/files/docker/production/nginx.conf:/etc/nginx/nginx.conf:ro")); + + let names: Vec<&str> = artifacts + .config_files + .iter() + .filter_map(|file| file.get("name").and_then(|name| name.as_str())) + .collect(); + assert!(names.contains(&"docker-compose.yml")); + assert!(names.contains(&".env")); + assert!(names.contains(&"nginx.conf")); + } + + #[test] + fn build_config_bundle_rejects_directory_mounts() { + let dir = TempDir::new().unwrap(); + let compose_dir = dir.path().join("docker/production"); + std::fs::create_dir_all(compose_dir.join("config")).unwrap(); + std::fs::write( + compose_dir.join("compose.yml"), + r#" +services: + api: + image: device-api:latest + volumes: + - ./config:/app/config:ro +"#, + ) + .unwrap(); + + let err = build_config_bundle( + dir.path(), + "production", + &compose_dir.join("compose.yml"), + None, + ) + .unwrap_err(); + + assert!( + err.to_string() + .contains("directory mounts are not supported"), + "unexpected error: {err}" + ); + } + + #[test] + fn artifact_metadata_marks_secret_like_files_hidden() { + let manifest = ConfigBundleManifest { + version: 1, + environment: "production".to_string(), + files: vec![ + ConfigBundleFile { + source_path: "docker/production/.env".to_string(), + destination_path: + "/opt/stacker/deployments/production/files/docker/production/.env" + .to_string(), + mode: "0644".to_string(), + size: 12, + sha256: "abc".to_string(), + }, + ConfigBundleFile { + source_path: "docker/production/nginx.conf".to_string(), + destination_path: + "/opt/stacker/deployments/production/files/docker/production/nginx.conf" + .to_string(), + mode: "0644".to_string(), + size: 10, + sha256: "def".to_string(), + }, + ], + }; + let artifacts = ConfigBundleArtifacts { + environment: "production".to_string(), + manifest_path: PathBuf::from(".stacker/deploy/production/config-bundle.manifest.json"), + archive_path: PathBuf::from(".stacker/deploy/production/config-bundle.tar.zst"), + remote_compose_path: PathBuf::from( + ".stacker/deploy/production/docker-compose.remote.yml", + ), + manifest, + config_files: vec![], + }; + + let metadata = artifacts.artifact_metadata(); + assert_eq!(metadata["environment"], "production"); + assert_eq!(metadata["config_files"][0]["content_hidden"], true); + assert_eq!(metadata["config_files"][1]["content_hidden"], false); + assert!(metadata["config_files"][0].get("content").is_none()); + } +} diff --git a/src/cli/config_parser.rs b/src/cli/config_parser.rs index 012e1449..3459114b 100644 --- a/src/cli/config_parser.rs +++ b/src/cli/config_parser.rs @@ -366,6 +366,9 @@ pub struct RegistryConfig { /// Per-target deployment profile in multi-target configs. #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct DeployProfileConfig { + #[serde(default)] + pub environment: Option, + #[serde(default)] pub compose_file: Option, @@ -401,6 +404,9 @@ pub struct DeployConfig { #[serde(default)] pub target: DeployTarget, + #[serde(default)] + pub environment: Option, + #[serde(default)] pub compose_file: Option, @@ -501,6 +507,10 @@ impl DeployConfig { Ok(DeployConfig { target: inferred_target, + environment: profile + .environment + .clone() + .or_else(|| self.environment.clone()), compose_file: profile.compose_file.clone(), deployment_hash: profile.deployment_hash.clone(), cloud: profile.cloud.clone(), @@ -512,6 +522,15 @@ impl DeployConfig { } } +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct EnvironmentConfig { + #[serde(default)] + pub compose_file: Option, + + #[serde(default)] + pub env_file: Option, +} + /// Cloud provider settings for cloud deployments. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CloudConfig { @@ -696,6 +715,9 @@ pub struct StackerConfig { #[serde(default)] pub deploy: DeployConfig, + #[serde(default)] + pub environments: BTreeMap, + #[serde(default)] pub ai: AiConfig, @@ -769,6 +791,42 @@ impl StackerConfig { Ok(config) } + pub fn selected_environment(&self, override_environment: Option<&str>) -> Option { + override_environment + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .or_else(|| self.deploy.environment.clone()) + } + + pub fn resolve_environment_config( + &self, + override_environment: Option<&str>, + ) -> Result, CliError> { + let Some(environment) = self.selected_environment(override_environment) else { + return Ok(None); + }; + + let configured = self.environments.get(&environment).cloned(); + let compose_file = configured + .as_ref() + .and_then(|config| config.compose_file.clone()) + .or_else(|| self.deploy.compose_file.clone()) + .or_else(|| Some(PathBuf::from(format!("docker/{environment}/compose.yml")))); + let env_file = configured + .as_ref() + .and_then(|config| config.env_file.clone()) + .or_else(|| self.env_file.clone()); + + Ok(Some(( + environment, + EnvironmentConfig { + compose_file, + env_file, + }, + ))) + } + /// Validate cross-field semantic constraints beyond serde deserialization. /// Returns a list of issues (errors, warnings, info). pub fn validate_semantics(&self) -> Vec { @@ -818,6 +876,7 @@ impl StackerConfig { Ok(target) => { let deploy = DeployConfig { target, + environment: profile.environment.clone(), compose_file: profile.compose_file.clone(), deployment_hash: profile.deployment_hash.clone(), cloud: profile.cloud.clone(), @@ -1291,6 +1350,7 @@ impl ConfigBuilder { proxy: self.proxy.unwrap_or_default(), deploy: DeployConfig { target: self.deploy_target.unwrap_or_default(), + environment: None, compose_file: None, deployment_hash: None, cloud: self.cloud, @@ -1299,6 +1359,7 @@ impl ConfigBuilder { default_target: None, targets: BTreeMap::new(), }, + environments: BTreeMap::new(), ai: self.ai.unwrap_or_default(), monitoring: self.monitoring.unwrap_or_default(), hooks: self.hooks.unwrap_or_default(), @@ -1425,6 +1486,7 @@ deploy: let resolved = config.with_resolved_deploy_target(None).unwrap(); assert_eq!(resolved.deploy.target, DeployTarget::Server); + assert!(resolved.deploy.environment.is_none()); assert_eq!( resolved .deploy @@ -1462,6 +1524,69 @@ deploy: assert!(resolved.deploy.compose_file.is_none()); } + #[test] + fn test_parse_environment_config_and_default_selection() { + let yaml = r#" +name: environment-app +app: + type: static +deploy: + target: cloud + environment: production +environments: + production: + compose_file: docker/production/compose.yml + env_file: docker/production/.env +"#; + + let config = StackerConfig::from_str(yaml).unwrap(); + assert_eq!(config.deploy.environment.as_deref(), Some("production")); + assert_eq!( + config + .environments + .get("production") + .and_then(|environment| environment.compose_file.as_ref()), + Some(&PathBuf::from("docker/production/compose.yml")) + ); + + let (environment, environment_config) = config + .resolve_environment_config(None) + .unwrap() + .expect("environment should resolve"); + assert_eq!(environment, "production"); + assert_eq!( + environment_config.compose_file, + Some(PathBuf::from("docker/production/compose.yml")) + ); + assert_eq!( + environment_config.env_file, + Some(PathBuf::from("docker/production/.env")) + ); + } + + #[test] + fn test_environment_override_uses_conventional_compose_path() { + let yaml = r#" +name: environment-app +app: + type: static +deploy: + target: cloud +"#; + + let config = StackerConfig::from_str(yaml).unwrap(); + let (environment, environment_config) = config + .resolve_environment_config(Some("staging")) + .unwrap() + .expect("environment should resolve"); + + assert_eq!(environment, "staging"); + assert_eq!( + environment_config.compose_file, + Some(PathBuf::from("docker/staging/compose.yml")) + ); + } + #[test] fn test_monitors_alias_for_monitoring() { let yaml = r#" @@ -1728,7 +1853,10 @@ app: let err = StackerConfig::from_str(yaml).unwrap_err(); let msg = err.to_string(); assert!(msg.contains("app.path"), "unexpected message: {msg}"); - assert!(msg.contains("quoted path string"), "unexpected message: {msg}"); + assert!( + msg.contains("quoted path string"), + "unexpected message: {msg}" + ); } #[test] diff --git a/src/cli/install_runner.rs b/src/cli/install_runner.rs index 7cddecd9..763f6750 100644 --- a/src/cli/install_runner.rs +++ b/src/cli/install_runner.rs @@ -98,6 +98,9 @@ pub struct DeployContext { /// Container runtime preference ("runc" or "kata"). pub runtime: String, + + /// Environment-specific config files collected from compose env_file and bind mounts. + pub config_bundle: Option, } impl DeployContext { @@ -511,7 +514,13 @@ impl DeployStrategy for CloudDeploy { // Step 1: Resolve or auto-create project eprintln!(" Resolving project '{}'...", project_name); - let project_body = stacker_client::build_project_body(config); + let mut project_body = stacker_client::build_project_body(config); + if let Some(bundle) = &context.config_bundle { + stacker_client::attach_config_bundle_to_project_body( + &mut project_body, + bundle, + ); + } let project = match client.find_project_by_name(&project_name).await? { Some(p) => { eprintln!(" Found project '{}' (id={}), syncing metadata...", p.name, p.id); @@ -669,6 +678,12 @@ impl DeployStrategy for CloudDeploy { // Step 4: Build deploy form let mut deploy_form = stacker_client::build_deploy_form(config); + if let Some(bundle) = &context.config_bundle { + stacker_client::attach_config_bundle_to_deploy_form( + &mut deploy_form, + bundle, + ); + } // Capture the server name from the form (auto-generated or overridden) // so we can persist it in the deployment lock even if the API fetch @@ -1055,7 +1070,15 @@ pub(crate) fn resolve_docker_registry_credentials( // Registry server: env var > config > default "docker.io" let server = first_non_empty_env(&["STACKER_DOCKER_REGISTRY", "DOCKER_REGISTRY"]) - .or_else(|| registry.and_then(|r| r.server.clone())); + .or_else(|| registry.and_then(|r| r.server.clone())) + .or_else(|| { + if username.is_some() || password.is_some() { + Some("docker.io".to_string()) + } else { + None + } + }) + .map(canonicalize_registry_server); if let Some(u) = username { creds.insert("docker_username".to_string(), serde_json::Value::String(u)); @@ -1070,6 +1093,27 @@ pub(crate) fn resolve_docker_registry_credentials( creds } +fn canonicalize_registry_server(server: String) -> String { + let trimmed = server.trim().trim_end_matches('/').to_string(); + let lower = trimmed.to_ascii_lowercase(); + + if lower == "docker.io" + || lower == "hub.docker.com" + || lower == "index.docker.io" + || lower == "registry-1.docker.io" + || lower == "https://docker.io" + || lower == "https://hub.docker.com" + || lower == "https://index.docker.io" + || lower == "https://index.docker.io/v1" + || lower == "https://index.docker.io/v1/" + || lower == "https://registry-1.docker.io" + { + "docker.io".to_string() + } else { + trimmed + } +} + #[allow(dead_code)] fn build_remote_deploy_payload(config: &StackerConfig) -> serde_json::Value { let cloud = config.deploy.cloud.as_ref(); @@ -1337,7 +1381,10 @@ impl DeployStrategy for ServerDeploy { .as_ref() .ok_or(CliError::ServerHostMissing)?; let project_name = resolve_remote_project_name(config, context); - let project_body = stacker_client::build_project_body(config); + let mut project_body = stacker_client::build_project_body(config); + if let Some(bundle) = &context.config_bundle { + stacker_client::attach_config_bundle_to_project_body(&mut project_body, bundle); + } let bootstrap_status_panel = true; let (response, effective_server_name) = tokio::runtime::Builder::new_current_thread() @@ -1406,6 +1453,9 @@ impl DeployStrategy for ServerDeploy { &effective_server_name, bootstrap_status_panel, ); + if let Some(bundle) = &context.config_bundle { + stacker_client::attach_config_bundle_to_deploy_form(&mut deploy_form, bundle); + } if let Some(server_obj) = deploy_form .get_mut("server") @@ -1595,7 +1645,7 @@ fn extract_server_ip(stdout: &str) -> Option { mod tests { use super::*; use crate::cli::config_parser::{ - CloudConfig, CloudOrchestrator, CloudProvider, ConfigBuilder, ServerConfig, + CloudConfig, CloudOrchestrator, CloudProvider, ConfigBuilder, RegistryConfig, ServerConfig, }; use std::sync::Mutex; @@ -1848,6 +1898,7 @@ mod tests { key_id_override: None, server_name_override: None, runtime: "runc".to_string(), + config_bundle: None, } } @@ -1974,6 +2025,7 @@ mod tests { key_id_override: None, server_name_override: None, runtime: "runc".to_string(), + config_bundle: None, }; assert_eq!(ctx.install_image(), "mycompany/install:v3"); } @@ -2114,6 +2166,49 @@ mod tests { ); } + #[test] + fn test_canonicalize_registry_server_maps_docker_hub_urls_to_docker_io() { + assert_eq!( + canonicalize_registry_server("https://index.docker.io/v1/".to_string()), + "docker.io" + ); + assert_eq!( + canonicalize_registry_server("https://registry-1.docker.io".to_string()), + "docker.io" + ); + assert_eq!( + canonicalize_registry_server("hub.docker.com".to_string()), + "docker.io" + ); + } + + #[test] + fn test_resolve_docker_registry_credentials_defaults_to_docker_io_when_auth_present() { + let config = ConfigBuilder::new() + .name("private-app") + .registry(RegistryConfig { + username: Some("syncopia-user".to_string()), + password: Some("secret".to_string()), + server: None, + }) + .build() + .unwrap(); + + let creds = resolve_docker_registry_credentials(&config); + assert_eq!( + creds.get("docker_username").and_then(|v| v.as_str()), + Some("syncopia-user") + ); + assert_eq!( + creds.get("docker_password").and_then(|v| v.as_str()), + Some("secret") + ); + assert_eq!( + creds.get("docker_registry").and_then(|v| v.as_str()), + Some("docker.io") + ); + } + #[test] fn test_cloud_deploy_runs_install_container() { let config = sample_cloud_config(); diff --git a/src/cli/mod.rs b/src/cli/mod.rs index f3fb66d2..0090e861 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -3,6 +3,7 @@ pub mod ai_field_matcher; pub mod ai_pipe_suggest; pub mod ai_scanner; pub mod ci_export; +pub mod config_bundle; pub mod config_parser; pub mod credentials; pub mod deployment_lock; diff --git a/src/cli/stacker_client.rs b/src/cli/stacker_client.rs index 977a0c2e..e7b00b22 100644 --- a/src/cli/stacker_client.rs +++ b/src/cli/stacker_client.rs @@ -2633,6 +2633,45 @@ pub fn build_project_body(config: &StackerConfig) -> serde_json::Value { }) } +pub fn attach_config_bundle_to_project_body( + project_body: &mut serde_json::Value, + artifacts: &crate::cli::config_bundle::ConfigBundleArtifacts, +) { + if let Some(custom) = project_body + .get_mut("custom") + .and_then(|custom| custom.as_object_mut()) + { + custom.insert( + "deployment_artifacts".to_string(), + serde_json::json!({ + "config_bundle": artifacts.artifact_metadata(), + }), + ); + } +} + +pub fn attach_config_bundle_to_deploy_form( + deploy_form: &mut serde_json::Value, + artifacts: &crate::cli::config_bundle::ConfigBundleArtifacts, +) { + if let Some(obj) = deploy_form.as_object_mut() { + obj.insert( + "environment".to_string(), + serde_json::Value::String(artifacts.environment.clone()), + ); + obj.insert( + "config_files".to_string(), + serde_json::Value::Array(artifacts.config_files.clone()), + ); + obj.insert( + "config_bundle".to_string(), + serde_json::json!({ + "manifest": artifacts.artifact_metadata(), + }), + ); + } +} + /// Build the deploy form payload that matches the Stacker server's /// `forms::project::Deploy` structure. /// Generate a deterministic but unique server name from the project name. @@ -2968,6 +3007,69 @@ mod tests { ); } + #[test] + fn test_attach_config_bundle_adds_deploy_files_and_stack_builder_metadata() { + let artifacts = crate::cli::config_bundle::ConfigBundleArtifacts { + environment: "production".to_string(), + manifest_path: std::path::PathBuf::from( + ".stacker/deploy/production/config-bundle.manifest.json", + ), + archive_path: std::path::PathBuf::from( + ".stacker/deploy/production/config-bundle.tar.zst", + ), + remote_compose_path: std::path::PathBuf::from( + ".stacker/deploy/production/docker-compose.remote.yml", + ), + manifest: crate::cli::config_bundle::ConfigBundleManifest { + version: 1, + environment: "production".to_string(), + files: vec![crate::cli::config_bundle::ConfigBundleFile { + source_path: "docker/production/.env".to_string(), + destination_path: + "/opt/stacker/deployments/production/files/docker/production/.env" + .to_string(), + mode: "0644".to_string(), + size: 17, + sha256: "abc123".to_string(), + }], + }, + config_files: vec![serde_json::json!({ + "name": ".env", + "content": "RUST_LOG=warning\n", + "destination_path": "/opt/stacker/deployments/production/files/docker/production/.env", + })], + }; + + let config = crate::cli::config_parser::ConfigBuilder::new() + .name("device-api") + .deploy_target(crate::cli::config_parser::DeployTarget::Cloud) + .build() + .unwrap(); + let mut project_body = build_project_body(&config); + let mut deploy_form = build_deploy_form(&config); + + attach_config_bundle_to_project_body(&mut project_body, &artifacts); + attach_config_bundle_to_deploy_form(&mut deploy_form, &artifacts); + + assert_eq!(deploy_form["environment"], "production"); + assert_eq!(deploy_form["config_files"][0]["name"], ".env"); + assert_eq!( + project_body["custom"]["deployment_artifacts"]["config_bundle"]["config_files"][0] + ["source_path"], + "docker/production/.env" + ); + assert_eq!( + project_body["custom"]["deployment_artifacts"]["config_bundle"]["config_files"][0] + ["content_hidden"], + true + ); + assert!( + project_body["custom"]["deployment_artifacts"]["config_bundle"]["config_files"][0] + .get("content") + .is_none() + ); + } + #[test] fn test_build_server_deploy_form_uses_existing_server_settings() { let config = crate::cli::config_parser::ConfigBuilder::new() diff --git a/src/console/commands/cli/config.rs b/src/console/commands/cli/config.rs index ee5015be..ecfdc6cd 100644 --- a/src/console/commands/cli/config.rs +++ b/src/console/commands/cli/config.rs @@ -99,7 +99,11 @@ fn load_raw_path_issues(path: &Path) -> Result, CliError> { Ok(issues) } -fn remove_empty_path_fields(value: &mut serde_yaml::Value, prefix: Option<&str>, applied: &mut Vec) { +fn remove_empty_path_fields( + value: &mut serde_yaml::Value, + prefix: Option<&str>, + applied: &mut Vec, +) { if let serde_yaml::Value::Mapping(map) = value { let keys_to_remove: Vec = map .iter() @@ -1147,7 +1151,9 @@ app: let issues = run_validate(&path).unwrap(); assert!(issues.iter().any(|issue| issue.contains("app.path"))); - assert!(issues.iter().any(|issue| issue.contains("quoted path string"))); + assert!(issues + .iter() + .any(|issue| issue.contains("quoted path string"))); } #[test] @@ -1271,7 +1277,9 @@ deploy: let applied = try_fix_raw_path_issues(&config_path).unwrap(); assert!(applied.iter().any(|item| item.contains("app.path"))); - assert!(applied.iter().any(|item| item.contains("deploy.server.ssh_key"))); + assert!(applied + .iter() + .any(|item| item.contains("deploy.server.ssh_key"))); let fixed = std::fs::read_to_string(&config_path).unwrap(); assert!(!fixed.contains("path: null")); @@ -1294,6 +1302,9 @@ app: let err = run_fix_interactive(&config_path).unwrap_err(); let msg = err.to_string(); assert!(msg.contains("app.path"), "unexpected message: {msg}"); - assert!(msg.contains("quoted path string"), "unexpected message: {msg}"); + assert!( + msg.contains("quoted path string"), + "unexpected message: {msg}" + ); } } diff --git a/src/console/commands/cli/deploy.rs b/src/console/commands/cli/deploy.rs index 778d1f40..703a3c61 100644 --- a/src/console/commands/cli/deploy.rs +++ b/src/console/commands/cli/deploy.rs @@ -5,6 +5,7 @@ use std::time::Duration; use crate::cli::ai_client::{ build_prompt, create_provider, ollama_complete_streaming, AiTask, PromptContext, }; +use crate::cli::config_bundle::build_config_bundle; use crate::cli::config_parser::{ AiProviderType, CloudConfig, CloudOrchestrator, CloudProvider, DeployTarget, ServerConfig, StackerConfig, @@ -478,15 +479,27 @@ fn validate_compose_for_deploy(compose_path: &Path) -> Result<(), CliError> { }; let services_key = serde_yaml::Value::String("services".to_string()); + let include_key = serde_yaml::Value::String("include".to_string()); let services = match root.get(&services_key) { - Some(serde_yaml::Value::Mapping(m)) => m, - _ => { - return Err(CliError::ConfigValidation( - "Compose file must define a top-level services mapping".to_string(), - )) - } + Some(serde_yaml::Value::Mapping(m)) => Some(m), + _ => None, }; + if services.is_none() { + match root.get(&include_key) { + Some(serde_yaml::Value::Sequence(_)) | Some(serde_yaml::Value::String(_)) => { + return Ok(()); + } + _ => { + return Err(CliError::ConfigValidation( + "Compose file must define a top-level services mapping".to_string(), + )) + } + } + } + + let services = services.expect("services checked above"); + let mut published_ports: std::collections::BTreeMap> = std::collections::BTreeMap::new(); @@ -866,6 +879,7 @@ fn prompt_select_cloud(access_token: &str) -> Result, + pub environment: Option, pub file: Option, pub dry_run: bool, pub force_rebuild: bool, @@ -897,6 +911,7 @@ impl DeployCommand { ) -> Self { Self { target, + environment: None, file, dry_run, force_rebuild, @@ -911,6 +926,11 @@ impl DeployCommand { } } + pub fn with_environment(mut self, environment: Option) -> Self { + self.environment = environment; + self + } + /// Builder method to set remote override flags from CLI args. pub fn with_remote_overrides( mut self, @@ -972,6 +992,7 @@ impl DeployCommand { } /// Parse a deploy target string into `DeployTarget`. +#[cfg(test)] fn parse_deploy_target(s: &str) -> Result { let json = format!("\"{}\"", s.to_lowercase()); serde_json::from_str::(&json).map_err(|_| { @@ -994,6 +1015,7 @@ pub struct RemoteDeployOverrides { /// Core deploy logic, extracted for testability. /// /// Takes injectable `CommandExecutor` so tests can mock shell calls. +#[allow(clippy::too_many_arguments)] pub fn run_deploy( project_dir: &Path, config_file: Option<&str>, @@ -1004,12 +1026,40 @@ pub fn run_deploy( executor: &dyn CommandExecutor, remote_overrides: &RemoteDeployOverrides, runtime: &str, +) -> Result { + run_deploy_for_environment( + project_dir, + config_file, + target_override, + None, + dry_run, + force_rebuild, + force_new, + executor, + remote_overrides, + runtime, + ) +} + +#[allow(clippy::too_many_arguments)] +pub fn run_deploy_for_environment( + project_dir: &Path, + config_file: Option<&str>, + target_override: Option<&str>, + environment_override: Option<&str>, + dry_run: bool, + force_rebuild: bool, + force_new: bool, + executor: &dyn CommandExecutor, + remote_overrides: &RemoteDeployOverrides, + runtime: &str, ) -> Result { let cred_manager = CredentialsManager::with_default_store(); run_deploy_with_credentials_manager( project_dir, config_file, target_override, + environment_override, dry_run, force_rebuild, force_new, @@ -1020,10 +1070,12 @@ pub fn run_deploy( ) } +#[allow(clippy::too_many_arguments)] fn run_deploy_with_credentials_manager( project_dir: &Path, config_file: Option<&str>, target_override: Option<&str>, + environment_override: Option<&str>, dry_run: bool, force_rebuild: bool, force_new: bool, @@ -1040,6 +1092,20 @@ fn run_deploy_with_credentials_manager( let mut config = StackerConfig::from_file(&config_path)?.with_resolved_deploy_target(target_override)?; + let selected_environment = if let Some((environment, environment_config)) = + config.resolve_environment_config(environment_override)? + { + config.deploy.environment = Some(environment.clone()); + if let Some(compose_file) = environment_config.compose_file { + config.deploy.compose_file = Some(compose_file); + } + if let Some(env_file) = environment_config.env_file { + config.env_file = Some(env_file); + } + Some(environment) + } else { + None + }; ensure_env_file_if_needed(&config, project_dir)?; // 2. Resolve deploy target/profile (flag > config default) @@ -1362,6 +1428,29 @@ fn run_deploy_with_credentials_manager( ); } eprintln!(" Compose file: {}", compose_path.display()); + if let Some(environment) = &selected_environment { + eprintln!( + " Environment: {} -> Target: {}", + environment, deploy_target + ); + } + + let config_bundle = if matches!(deploy_target, DeployTarget::Cloud | DeployTarget::Server) { + if let Some(environment) = selected_environment.as_deref() { + let bundle = build_config_bundle( + project_dir, + environment, + &compose_path, + config.env_file.as_deref(), + )?; + eprintln!(" Config bundle: {}", bundle.archive_path.display()); + Some(bundle) + } else { + None + } + } else { + None + }; // 5c. Report hooks (dry-run) if dry_run { @@ -1386,6 +1475,7 @@ fn run_deploy_with_credentials_manager( key_id_override: remote_overrides.key_id, server_name_override: remote_overrides.server_name.clone().or(lock_server_name), runtime: runtime.to_string(), + config_bundle, }; let result = strategy.deploy(&config, &context, executor)?; @@ -1409,10 +1499,11 @@ impl CallableTrait for DeployCommand { // ── Spinner while deploying ────────────────── let spin = progress::deploy_spinner("starting..."); - let result = run_deploy( + let result = run_deploy_for_environment( &project_dir, self.file.as_deref(), self.target.as_deref(), + self.environment.as_deref(), self.dry_run, self.force_rebuild, self.force_new, @@ -2055,10 +2146,6 @@ mod tests { }, } } - - fn recorded_calls(&self) -> Vec<(String, Vec)> { - self.calls.lock().unwrap().clone() - } } impl CommandExecutor for MockExecutor { @@ -2212,6 +2299,50 @@ mod tests { assert!(result.is_ok()); } + #[test] + fn test_deploy_environment_override_uses_environment_compose() { + let config = r#" +name: device-api +app: + type: static + path: . +deploy: + target: local +"#; + let compose = r#" +services: + api: + image: device-api:latest + environment: + RUST_LOG: warning +"#; + let dir = setup_local_project(&[ + ("index.html", "

hello

"), + ("stacker.yml", config), + ("docker/production/compose.yml", compose), + ]); + let executor = MockExecutor::success(); + + let result = run_deploy_for_environment( + dir.path(), + None, + Some("local"), + Some("production"), + true, + false, + false, + &executor, + &RemoteDeployOverrides::default(), + "runc", + ); + + assert!(result.is_ok()); + assert!( + !dir.path().join(".stacker/docker-compose.yml").exists(), + "environment compose should be used instead of generating .stacker/docker-compose.yml" + ); + } + #[test] fn test_deploy_local_with_image_skips_build() { let config = "name: test-app\napp:\n type: static\n path: .\n image: nginx:latest\n"; @@ -2246,6 +2377,7 @@ mod tests { dir.path(), None, None, + None, true, false, false, @@ -2585,6 +2717,20 @@ services: assert!(msg.contains("nginx_proxy_manager")); } + #[test] + fn test_validate_compose_for_deploy_allows_include_only_compose() { + let dir = TempDir::new().unwrap(); + let compose_path = dir.path().join("docker-compose.yml"); + let compose = r#" +include: + - ../../postgres/docker/local/compose.yml + - ../../website/docker/local/compose.yml +"#; + std::fs::write(&compose_path, compose).unwrap(); + + validate_compose_for_deploy(&compose_path).unwrap(); + } + #[test] fn test_parse_deploy_target_valid() { assert_eq!(parse_deploy_target("local").unwrap(), DeployTarget::Local); diff --git a/src/console/commands/cli/init.rs b/src/console/commands/cli/init.rs index ebbf0565..5c065c6f 100644 --- a/src/console/commands/cli/init.rs +++ b/src/console/commands/cli/init.rs @@ -98,7 +98,7 @@ pub fn full_config_reference_example() -> &'static str { # # registry: # Docker registry credentials for private images # # username: "${DOCKER_USERNAME}" # # password: "${DOCKER_PASSWORD}" -# # server: "https://index.docker.io/v1/" # optional, defaults to Docker Hub +# # server: "docker.io" # optional, defaults to Docker Hub # # ai: # enabled: true diff --git a/src/forms/project/custom.rs b/src/forms/project/custom.rs index f05dfc68..c33d0d26 100644 --- a/src/forms/project/custom.rs +++ b/src/forms/project/custom.rs @@ -40,6 +40,8 @@ pub struct Custom { pub marketplace_seed_jobs: JsonValue, #[serde(default)] pub marketplace_post_deploy_hooks: JsonValue, + #[serde(default)] + pub deployment_artifacts: JsonValue, #[serde(flatten)] pub networks: forms::project::ComposeNetworks, // all networks } @@ -141,6 +143,11 @@ mod tests { "marketplace_update_mode_capabilities": { "mode_self_managed": true, "mode_managed_status_panel": true + }, + "deployment_artifacts": { + "config_bundle": { + "environment": "production" + } } })) .expect("custom form should deserialize"); @@ -156,5 +163,9 @@ mod tests { serialized["marketplace_update_mode_capabilities"]["mode_managed_status_panel"], json!(true) ); + assert_eq!( + serialized["deployment_artifacts"]["config_bundle"]["environment"], + json!("production") + ); } } diff --git a/src/forms/project/deploy.rs b/src/forms/project/deploy.rs index 88c7a712..9660522c 100644 --- a/src/forms/project/deploy.rs +++ b/src/forms/project/deploy.rs @@ -34,13 +34,13 @@ fn validate_cloud_instance_config(deploy: &Deploy) -> Result<(), serde_valid::va let mut missing = Vec::new(); - if deploy.server.region.as_ref().map_or(true, |s| s.is_empty()) { + if deploy.server.region.as_ref().is_none_or(|s| s.is_empty()) { missing.push("region"); } - if deploy.server.server.as_ref().map_or(true, |s| s.is_empty()) { + if deploy.server.server.as_ref().is_none_or(|s| s.is_empty()) { missing.push("server"); } - if deploy.server.os.as_ref().map_or(true, |s| s.is_empty()) { + if deploy.server.os.as_ref().is_none_or(|s| s.is_empty()) { missing.push("os"); } @@ -54,7 +54,7 @@ fn validate_cloud_instance_config(deploy: &Deploy) -> Result<(), serde_valid::va } } -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize, Validate)] +#[derive(Default, Clone, PartialEq, Serialize, Deserialize, Validate)] #[validate(custom(validate_cloud_instance_config))] pub struct Deploy { #[validate] @@ -66,6 +66,29 @@ pub struct Deploy { /// Optional Docker registry credentials for pulling private images. #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) registry: Option, + /// Optional selected deploy environment, e.g. development/staging/production. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) environment: Option, + /// Config files uploaded by the CLI. Contents may include secrets and must not be logged. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) config_files: Option, + /// Safe metadata for Stack Builder artifact/config-file visibility. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) config_bundle: Option, +} + +impl std::fmt::Debug for Deploy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Deploy") + .field("stack", &self.stack) + .field("server", &self.server) + .field("cloud", &self.cloud) + .field("registry", &self.registry) + .field("environment", &self.environment) + .field("config_files", &"[REDACTED]") + .field("config_bundle", &self.config_bundle) + .finish() + } } #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize, Validate)] diff --git a/src/forms/project/payload.rs b/src/forms/project/payload.rs index 6a6b3234..174e879e 100644 --- a/src/forms/project/payload.rs +++ b/src/forms/project/payload.rs @@ -1,10 +1,11 @@ use crate::forms; use crate::models; use serde::{Deserialize, Serialize}; +use serde_json::Value; use serde_valid::Validate; use std::convert::TryFrom; -#[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize, Validate)] +#[derive(Default, Clone, PartialEq, Serialize, Deserialize, Validate)] #[serde(rename_all = "snake_case")] pub struct Payload { pub(crate) id: Option, @@ -23,10 +24,41 @@ pub struct Payload { /// Docker registry credentials for pulling private images on the target server. #[serde(default, skip_serializing_if = "Option::is_none")] pub registry: Option, + /// Optional selected deploy environment, e.g. development/staging/production. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub environment: Option, + /// Deploy-time config files uploaded by the CLI. Contents may include secrets. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub config_files: Option, + /// Safe metadata for the deploy-time config bundle. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub config_bundle: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub runtime_artifact_bundle: Option, } +impl std::fmt::Debug for Payload { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Payload") + .field("id", &self.id) + .field("project_id", &self.project_id) + .field("deployment_hash", &self.deployment_hash) + .field("user_token", &self.user_token) + .field("user_email", &self.user_email) + .field("cloud", &self.cloud) + .field("server", &self.server) + .field("stack", &self.stack) + .field("custom", &"[REDACTED]") + .field("docker_compose", &"[REDACTED]") + .field("registry", &self.registry) + .field("environment", &self.environment) + .field("config_files", &"[REDACTED]") + .field("config_bundle", &self.config_bundle) + .field("runtime_artifact_bundle", &"[REDACTED]") + .finish() + } +} + impl TryFrom<&models::Project> for Payload { type Error = String; @@ -69,7 +101,29 @@ mod tests { ], "marketplace_post_deploy_hooks": [ {"name": "notify"} - ] + ], + "deployment_artifacts": { + "config_bundle": { + "remote_compose_path": ".stacker/deploy/production/docker-compose.remote.yml" + } + } + }, + "environment": "production", + "config_files": [ + { + "name": "docker-compose.yml", + "content": "services:\n app:\n image: example/app:1.0.0\n" + } + ], + "config_bundle": { + "manifest": { + "environment": "production", + "config_files": [ + { + "destination_path": "/opt/app/.env" + } + ] + } }, "runtime_artifact_bundle": { "filename": "runtime-bundle.tgz", @@ -100,11 +154,24 @@ mod tests { custom["marketplace_post_deploy_hooks"][0]["name"], json!("notify") ); + assert_eq!( + custom["deployment_artifacts"]["config_bundle"]["remote_compose_path"], + json!(".stacker/deploy/production/docker-compose.remote.yml") + ); assert_eq!( payload .runtime_artifact_bundle .expect("runtime bundle should exist")["download_url"], json!("https://objects.trydirect.test/runtime-bundle.tgz") ); + assert_eq!(payload.environment, Some("production".to_string())); + assert_eq!( + payload.config_files.expect("config files should exist")[0]["name"], + json!("docker-compose.yml") + ); + assert_eq!( + payload.config_bundle.expect("config bundle should exist")["manifest"]["environment"], + json!("production") + ); } } diff --git a/src/routes/project/deploy.rs b/src/routes/project/deploy.rs index a399c321..d8d1aa7a 100644 --- a/src/routes/project/deploy.rs +++ b/src/routes/project/deploy.rs @@ -535,6 +535,126 @@ fn sync_runtime_artifact_bundle( Ok(()) } +fn upsert_root_field(target: &mut serde_json::Value, field: &str, value: &serde_json::Value) { + ensure_root_object(target).insert(field.to_string(), value.clone()); +} + +fn upsert_deployment_artifact( + target: &mut serde_json::Value, + field: &str, + value: &serde_json::Value, +) { + let custom = ensure_custom_object(target); + let deployment_artifacts = custom + .entry("deployment_artifacts".to_string()) + .or_insert_with(|| serde_json::json!({})); + if !deployment_artifacts.is_object() { + *deployment_artifacts = serde_json::json!({}); + } + + deployment_artifacts + .as_object_mut() + .expect("deployment_artifacts should be normalized to an object") + .insert(field.to_string(), value.clone()); +} + +fn basename_from_path(path: &str) -> Option<&str> { + Path::new(path) + .file_name() + .and_then(|name| name.to_str()) + .map(str::trim) + .filter(|name| !name.is_empty()) +} + +fn compose_content_from_config_files( + config_files: &serde_json::Value, +) -> Result, String> { + let files = config_files + .as_array() + .ok_or_else(|| "config_files must be an array".to_string())?; + + for file in files { + let file_name = file + .get("name") + .and_then(|value| value.as_str()) + .map(str::trim) + .filter(|name| !name.is_empty()) + .or_else(|| { + file.get("destination_path") + .and_then(|value| value.as_str()) + .and_then(basename_from_path) + }); + + if let Some(file_name) = file_name { + if crate::project_app::is_compose_filename(file_name) { + let content = file + .get("content") + .and_then(|value| value.as_str()) + .ok_or_else(|| { + format!( + "compose config file '{}' is missing string content", + file_name + ) + })?; + return Ok(Some(content.to_string())); + } + } + } + + Ok(None) +} + +fn apply_deploy_bundle( + project: &mut models::Project, + form: &forms::project::Deploy, +) -> Result, String> { + if let Some(environment) = form + .environment + .as_ref() + .map(|value| value.trim()) + .filter(|value| !value.is_empty()) + { + let environment_value = serde_json::Value::String(environment.to_string()); + upsert_root_field(&mut project.metadata, "environment", &environment_value); + upsert_root_field(&mut project.request_json, "environment", &environment_value); + } + + let compose_content = match form + .config_files + .as_ref() + .filter(|value| is_non_empty_json(value)) + { + Some(config_files) => { + upsert_root_field(&mut project.metadata, "config_files", config_files); + upsert_root_field(&mut project.request_json, "config_files", config_files); + compose_content_from_config_files(config_files)? + } + None => None, + }; + + if let Some(config_bundle) = form + .config_bundle + .as_ref() + .filter(|value| is_non_empty_json(value)) + { + upsert_root_field(&mut project.metadata, "config_bundle", config_bundle); + upsert_root_field(&mut project.request_json, "config_bundle", config_bundle); + + let artifact_metadata = config_bundle + .get("manifest") + .cloned() + .unwrap_or_else(|| config_bundle.clone()); + upsert_deployment_artifact(&mut project.metadata, "config_bundle", &artifact_metadata); + upsert_deployment_artifact( + &mut project.request_json, + "config_bundle", + &artifact_metadata, + ); + } + + Ok(compose_content) +} + async fn load_project_template_version( pg_pool: &PgPool, project: &models::Project, @@ -567,6 +687,8 @@ async fn execute_deployment( cloud: models::Cloud, server: models::Server, ) -> Result<(i32, i32)> { + let deploy_compose = apply_deploy_bundle(&mut project, form) + .map_err(|err| JsonResponse::::build().bad_request(err))?; let template_version = load_project_template_version(pg_pool, &project) .await .map_err(|err| JsonResponse::::build().internal_server_error(err))?; @@ -577,9 +699,12 @@ async fn execute_deployment( let id = project.id; let dc = DcBuilder::new(project); - let fc = dc - .build() - .map_err(|err| JsonResponse::::build().internal_server_error(err))?; + let fc = match deploy_compose { + Some(compose) => compose, + None => dc + .build() + .map_err(|err| JsonResponse::::build().internal_server_error(err))?, + }; let mut new_public_key: Option = None; let mut bootstrap_private_key: Option = None; @@ -1311,9 +1436,10 @@ pub async fn rollback( #[cfg(test)] mod tests { use super::{ - build_runtime_artifact_bundle, preserve_marketplace_runtime_artifacts, - resolve_provided_ssh_keypair, sync_runtime_artifact_bundle, validate_min_cpu_requirement, - validate_min_disk_requirement, validate_min_ram_requirement, + apply_deploy_bundle, build_runtime_artifact_bundle, compose_content_from_config_files, + preserve_marketplace_runtime_artifacts, resolve_provided_ssh_keypair, + sync_runtime_artifact_bundle, validate_min_cpu_requirement, validate_min_disk_requirement, + validate_min_ram_requirement, }; use crate::configuration::Settings; use crate::connectors::app_service_catalog::ServerCapacity; @@ -1462,6 +1588,91 @@ mod tests { assert!(err.contains("2")); } + #[test] + fn compose_content_from_config_files_prefers_uploaded_compose() { + let compose = compose_content_from_config_files(&json!([ + { + "name": ".env", + "content": "APP_ENV=production" + }, + { + "name": "docker-compose.yml", + "content": "services:\n website:\n image: syncopiaapp/website:latest\n" + } + ])) + .expect("config files should be valid") + .expect("compose should be discovered"); + + assert!(compose.contains("syncopiaapp/website:latest")); + } + + #[test] + fn apply_deploy_bundle_merges_runtime_fields_and_returns_compose() { + let mut project = models::Project::new( + "user-1".to_string(), + "syncopia".to_string(), + json!({ + "custom": { + "web": [], + "custom_stack_code": "syncopia" + } + }), + json!({ + "custom": { + "web": [], + "custom_stack_code": "syncopia" + } + }), + ); + let form = forms::project::Deploy { + environment: Some("prod".to_string()), + config_files: Some(json!([ + { + "name": "docker-compose.yml", + "content": "services:\n website:\n image: syncopiaapp/website:latest\n", + "destination_path": "docker-compose.yml" + }, + { + "name": ".env", + "content": "WEBSITE_IMAGE=syncopiaapp/website:latest\n", + "destination_path": "/opt/stacker/deployments/prod/files/.env" + } + ])), + config_bundle: Some(json!({ + "manifest": { + "environment": "prod", + "config_files": [ + { + "destination_path": "/opt/stacker/deployments/prod/files/.env" + } + ] + } + })), + ..Default::default() + }; + + let compose = apply_deploy_bundle(&mut project, &form) + .expect("bundle application should succeed") + .expect("compose should be available"); + + assert!(compose.contains("syncopiaapp/website:latest")); + assert_eq!(project.metadata["environment"], json!("prod")); + assert_eq!(project.request_json["environment"], json!("prod")); + assert_eq!( + project.metadata["config_files"][0]["name"], + json!("docker-compose.yml") + ); + assert_eq!( + project.metadata["custom"]["deployment_artifacts"]["config_bundle"]["environment"], + json!("prod") + ); + assert_eq!( + project.request_json["custom"]["deployment_artifacts"]["config_bundle"]["config_files"] + [0]["destination_path"], + json!("/opt/stacker/deployments/prod/files/.env") + ); + } + #[test] fn preserve_marketplace_runtime_artifacts_backfills_from_request_json_and_version() { let mut project = models::Project::new( diff --git a/website/stacker.yml b/website/stacker.yml index 47dd7029..cb997526 100644 --- a/website/stacker.yml +++ b/website/stacker.yml @@ -53,7 +53,7 @@ deploy: registry: username: trydirect password: ${DOCKER_PASSWORD} - server: https://index.docker.io/v1/ + server: docker.io ai: enabled: true provider: ollama @@ -75,4 +75,4 @@ hooks: post_deploy: scripts/post-deploy.sh on_failure: scripts/on-failure.sh env_file: .env -env: {} \ No newline at end of file +env: {}