From 9e9f2dc2f853e1b3dafcde7dc9a0d7af2babebf4 Mon Sep 17 00:00:00 2001 From: Mikhail Shchatko Date: Mon, 1 Aug 2022 13:07:07 +0300 Subject: [PATCH] Generate tasks separately for Windows, MacOS, Linux distro groups --- CHANGELOG.md | 4 ++ Cargo.lock | 2 +- Cargo.toml | 2 +- src/evergreen/evg_config_utils.rs | 66 ++++++++++++++++++++++++- src/evergreen_names.rs | 8 +++ src/lib.rs | 57 ++++++++++++++------- src/services/config_extraction.rs | 7 +++ src/task_types/burn_in_tests.rs | 4 +- src/task_types/fuzzer_tasks.rs | 3 ++ src/task_types/resmoke_config_writer.rs | 16 +++++- src/task_types/resmoke_tasks.rs | 11 +++++ src/utils/task_name.rs | 48 +++++++++++++----- tests/integration_test.rs | 2 +- 13 files changed, 192 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39e2668..f4b846d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.5.0 - 2022-08-01 + +* Generate tasks separately for Windows, MacOS, Linux distro groups. + ## 0.4.7 - 2022-07-14 * Add support for burn_in_tags generation. diff --git a/Cargo.lock b/Cargo.lock index ee58a6e..1f8b9ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1219,7 +1219,7 @@ dependencies = [ [[package]] name = "mongo-task-generator" -version = "0.4.7" +version = "0.5.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index f9a6f38..9dc6553 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mongo-task-generator" -version = "0.4.7" +version = "0.5.0" repository = "https://github.com/mongodb/mongo-task-generator" authors = ["Decision Automation Group "] edition = "2018" diff --git a/src/evergreen/evg_config_utils.rs b/src/evergreen/evg_config_utils.rs index ef91b8c..f608bbe 100644 --- a/src/evergreen/evg_config_utils.rs +++ b/src/evergreen/evg_config_utils.rs @@ -1,4 +1,5 @@ use std::collections::{HashMap, HashSet}; +use std::vec; use anyhow::{bail, Result}; use lazy_static::lazy_static; @@ -7,7 +8,9 @@ use shrub_rs::models::commands::EvgCommand::Function; use shrub_rs::models::params::ParamValue; use shrub_rs::models::{commands::FunctionCall, task::EvgTask, variant::BuildVariant}; -use crate::evergreen_names::{ENTERPRISE_MODULE, GENERATE_RESMOKE_TASKS, IS_FUZZER}; +use crate::evergreen_names::{ + ENTERPRISE_MODULE, GENERATE_RESMOKE_TASKS, IS_FUZZER, LINUX, MACOS, WINDOWS, +}; use crate::utils::task_name::remove_gen_suffix; lazy_static! { @@ -229,6 +232,17 @@ pub trait EvgConfigUtils: Sync + Send { /// /// true if given build variant includes the enterprise module. fn is_enterprise_build_variant(&self, build_variant: &BuildVariant) -> bool; + + /// Infer platform that build variant will be running on. + /// + /// # Arguments + /// + /// * `build_variant` - Build variant to query. + /// + /// # Returns + /// + /// Linux, or Mac, or Windows platform that build variant will be running on. + fn infer_build_variant_platform(&self, build_variant: &BuildVariant) -> String; } /// Service for utilities to help interpret evergreen configuration. @@ -581,6 +595,33 @@ impl EvgConfigUtils for EvgConfigUtilsImpl { false } } + + /// Infer platform that build variant will run on. + /// + /// # Arguments + /// + /// * `build_variant` - Build variant to query. + /// + /// # Returns + /// + /// linux, or mac, or windows platform that build variant will run on. + fn infer_build_variant_platform(&self, build_variant: &BuildVariant) -> String { + let distro = build_variant + .run_on + .as_ref() + .unwrap_or(&vec!["".to_string()]) + .first() + .unwrap_or(&"".to_string()) + .to_lowercase(); + + if distro.contains(MACOS) { + MACOS.to_string() + } else if distro.contains(WINDOWS) { + WINDOWS.to_string() + } else { + LINUX.to_string() + } + } } /// Get the shrub function make the 'generate resmoke task' call in the given task. @@ -1271,4 +1312,27 @@ mod tests { assert!(!evg_config_utils.is_enterprise_build_variant(&build_variant)); } + + // tests for infer_build_variant_platform + #[rstest] + #[case(Some(vec!["rhel80-small".to_string()]), "linux".to_string())] + #[case(Some(vec!["windows-vsCurrent-small".to_string()]), "windows".to_string())] + #[case(Some(vec!["macos-1100".to_string()]), "macos".to_string())] + #[case(Some(vec!["rhel80-small".to_string(), "macos-1100".to_string()]), "linux".to_string())] + #[case(Some(vec![]), "linux".to_string())] + fn test_infer_build_variant_platform( + #[case] distros: Option>, + #[case] platform: String, + ) { + let build_variant = BuildVariant { + run_on: distros, + ..Default::default() + }; + let evg_config_utils = EvgConfigUtilsImpl::new(); + + assert_eq!( + evg_config_utils.infer_build_variant_platform(&build_variant), + platform + ); + } } diff --git a/src/evergreen_names.rs b/src/evergreen_names.rs index 6c1233c..f4e66db 100644 --- a/src/evergreen_names.rs +++ b/src/evergreen_names.rs @@ -111,3 +111,11 @@ pub const MULTIVERSION_LAST_CONTINUOUS: &str = "last_continuous"; // Distros /// Name of compile task distro for burn_in_tags. pub const COMPILE_TASK_DISTRO: &str = "rhel80-xlarge"; + +// Distro group names +/// Windows distro group name. +pub const WINDOWS: &str = "windows"; +/// MacOS distro group name. +pub const MACOS: &str = "macos"; +/// Linux distro group name. +pub const LINUX: &str = "linux"; diff --git a/src/lib.rs b/src/lib.rs index 1d2953e..21fc116 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -461,6 +461,9 @@ impl GenerateTasksService for GenerateTasksServiceImpl { let is_enterprise = self .evg_config_utils .is_enterprise_build_variant(build_variant); + let platform = self + .evg_config_utils + .infer_build_variant_platform(build_variant); for task in &build_variant.tasks { // Burn in tasks could be different for each build variant, so we will always // handle them. @@ -502,7 +505,7 @@ impl GenerateTasksService for GenerateTasksServiceImpl { } // Skip tasks that have already been seen. - let task_name = lookup_task_name(is_enterprise, &task.name); + let task_name = lookup_task_name(is_enterprise, &task.name, &platform); if seen_tasks.contains(&task_name) { continue; } @@ -553,13 +556,24 @@ impl GenerateTasksService for GenerateTasksServiceImpl { Some(self.gen_fuzzer_service.generate_fuzzer_task(¶ms)?) } else { - event!(Level::INFO, "Generating resmoke task: {}", task_def.name); let is_enterprise = self .evg_config_utils .is_enterprise_build_variant(build_variant); - let params = self - .config_extraction_service - .task_def_to_resmoke_params(task_def, is_enterprise)?; + let platform = self + .evg_config_utils + .infer_build_variant_platform(build_variant); + event!( + Level::INFO, + "Generating resmoke task: {}, is_enterprise: {}, platform: {}", + task_def.name, + is_enterprise, + platform + ); + let params = self.config_extraction_service.task_def_to_resmoke_params( + task_def, + is_enterprise, + Some(platform), + )?; Some( self.gen_resmoke_service .generate_resmoke_task(¶ms, &build_variant.name) @@ -593,6 +607,9 @@ impl GenerateTasksService for GenerateTasksServiceImpl { let is_enterprise = self .evg_config_utils .is_enterprise_build_variant(build_variant); + let platform = self + .evg_config_utils + .infer_build_variant_platform(build_variant); let mut gen_config = GeneratedConfig::new(); let mut generating_tasks = vec![]; let large_distro_name = self @@ -617,7 +634,7 @@ impl GenerateTasksService for GenerateTasksServiceImpl { let task_name = if task.name == BURN_IN_TESTS { format!("{}-{}", BURN_IN_PREFIX, build_variant.name) } else { - lookup_task_name(is_enterprise, &task.name) + lookup_task_name(is_enterprise, &task.name, &platform) }; if let Some(generated_task) = generated_tasks.get(&task_name) { @@ -687,20 +704,17 @@ impl GenerateTasksService for GenerateTasksServiceImpl { /// # Arguments /// /// * `is_enterprise` - Whether the task is for an enterprise build variant. -/// * `task` - Evergreen definition of task. +/// * `task` - Name of task. +/// * `platform` - Platform that task will run on. /// /// # Returns /// /// Name to use for task. -fn lookup_task_name(is_enterprise: bool, task_name: &str) -> String { - if task_name == BURN_IN_TESTS { - return task_name.to_string(); - } - +fn lookup_task_name(is_enterprise: bool, task_name: &str, platform: &str) -> String { if is_enterprise { - format!("{}-{}", task_name, ENTERPRISE_MODULE) + format!("{}-{}-{}", task_name, platform, ENTERPRISE_MODULE) } else { - task_name.to_string() + format!("{}-{}", task_name, platform) } } @@ -735,7 +749,8 @@ fn create_task_worker( .unwrap(); let is_enterprise = evg_config_utils.is_enterprise_build_variant(&build_variant); - let task_name = lookup_task_name(is_enterprise, &task_def.name); + let platform = evg_config_utils.infer_build_variant_platform(&build_variant); + let task_name = lookup_task_name(is_enterprise, &task_def.name, &platform); if let Some(generated_task) = generated_task { let mut generated_tasks = generated_tasks.lock().unwrap(); @@ -933,15 +948,16 @@ mod tests { // tests for lookup_task_name. #[rstest] - #[case(false, "my_task", "my_task")] - #[case(true, "my_task", "my_task-enterprise")] + #[case(false, "my_task", "my_platform", "my_task-my_platform")] + #[case(true, "my_task", "my_platform", "my_task-my_platform-enterprise")] fn test_lookup_task_name_should_use_enterprise_when_specified( #[case] is_enterprise: bool, #[case] task_name: &str, + #[case] platform: &str, #[case] expected_task_name: &str, ) { assert_eq!( - lookup_task_name(is_enterprise, task_name), + lookup_task_name(is_enterprise, task_name, platform), expected_task_name.to_string() ); } @@ -1048,6 +1064,10 @@ mod tests { fn is_enterprise_build_variant(&self, _build_variant: &BuildVariant) -> bool { todo!() } + + fn infer_build_variant_platform(&self, _build_variant: &BuildVariant) -> String { + todo!() + } } struct MockResmokeConfigActorService {} @@ -1083,6 +1103,7 @@ mod tests { &self, _task_def: &EvgTask, _is_enterprise: bool, + _platform: Option, ) -> Result { todo!() } diff --git a/src/services/config_extraction.rs b/src/services/config_extraction.rs index 5a2d1c8..d1dee35 100644 --- a/src/services/config_extraction.rs +++ b/src/services/config_extraction.rs @@ -37,6 +37,8 @@ pub trait ConfigExtractionService: Sync + Send { /// # Arguments /// /// * `task-def` - Task definition of task to generate. + /// * `is_enterprise` - Is this being generated for an enterprise build variant. + /// * `platform` - Platform that task will run on. /// /// # Returns /// @@ -45,6 +47,7 @@ pub trait ConfigExtractionService: Sync + Send { &self, task_def: &EvgTask, is_enterprise: bool, + platform: Option, ) -> Result; } @@ -160,6 +163,7 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl { config_location: self.config_location.clone(), dependencies: self.determine_task_dependencies(task_def), is_enterprise, + platform: Some(evg_config_utils.infer_build_variant_platform(build_variant)), }) } @@ -169,6 +173,7 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl { /// /// * `task_def` - Task definition of task to generate. /// * `is_enterprise` - Is this being generated for an enterprise build variant. + /// * `platform` - Platform that task will run on. /// /// # Returns /// @@ -177,6 +182,7 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl { &self, task_def: &EvgTask, is_enterprise: bool, + platform: Option, ) -> Result { let task_name = remove_gen_suffix(&task_def.name).to_string(); let suite = self.evg_config_utils.find_suite_name(task_def).to_string(); @@ -209,6 +215,7 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl { dependencies: self.determine_task_dependencies(task_def), is_enterprise, pass_through_vars: self.evg_config_utils.get_gen_task_vars(task_def), + platform, }) } } diff --git a/src/task_types/burn_in_tests.rs b/src/task_types/burn_in_tests.rs index a3a0f64..dfff2eb 100644 --- a/src/task_types/burn_in_tests.rs +++ b/src/task_types/burn_in_tests.rs @@ -169,7 +169,7 @@ impl BurnInServiceImpl { for (index, test) in discovered_task.test_list.iter().enumerate() { let mut params = self .config_extraction_service - .task_def_to_resmoke_params(task_def, false)?; + .task_def_to_resmoke_params(task_def, false, None)?; update_resmoke_params_for_burn_in(&mut params, test); if params.generate_multiversion_combos { @@ -238,6 +238,7 @@ impl BurnInServiceImpl { origin_suite: origin_suite.clone(), mv_exclude_tags: suite_info.multiversion_tags.clone(), is_enterprise: false, + platform: None, }; self.gen_resmoke_task_service.build_resmoke_sub_task( @@ -513,6 +514,7 @@ mod tests { &self, _task_def: &EvgTask, _is_enterprise: bool, + _platform: Option, ) -> Result { Ok(ResmokeGenParams { generate_multiversion_combos: self.is_multiversion, diff --git a/src/task_types/fuzzer_tasks.rs b/src/task_types/fuzzer_tasks.rs index c8f11a6..a98ff8e 100644 --- a/src/task_types/fuzzer_tasks.rs +++ b/src/task_types/fuzzer_tasks.rs @@ -57,6 +57,8 @@ pub struct FuzzerGenTaskParams { pub dependencies: Vec, /// Is this task for enterprise builds. pub is_enterprise: bool, + /// Name of platform the task will run on. + pub platform: Option, } impl FuzzerGenTaskParams { @@ -273,6 +275,7 @@ fn build_fuzzer_sub_task( Some(sub_task_index), params.num_tasks as usize, params.is_enterprise, + params.platform.as_deref(), ); let mut commands = vec![]; diff --git a/src/task_types/resmoke_config_writer.rs b/src/task_types/resmoke_config_writer.rs index 1a4f958..c91e35f 100644 --- a/src/task_types/resmoke_config_writer.rs +++ b/src/task_types/resmoke_config_writer.rs @@ -148,7 +148,13 @@ impl WriteConfigActorImpl { let filename = format!( "{}.yml", - name_generated_task(&s.name, s.index, total_tasks, s.is_enterprise) + name_generated_task( + &s.name, + s.index, + total_tasks, + s.is_enterprise, + s.platform.as_deref() + ) ); let mut path = PathBuf::from(&self.target_dir); path.push(filename); @@ -182,7 +188,13 @@ impl WriteConfigActorImpl { let misc_config = origin_config.with_new_tests(None, Some(&test_list)); let filename = format!( "{}.yml", - name_generated_task(&s.name, s.index, total_tasks, s.is_enterprise) + name_generated_task( + &s.name, + s.index, + total_tasks, + s.is_enterprise, + s.platform.as_deref() + ) ); let mut path = PathBuf::from(&self.target_dir); path.push(filename); diff --git a/src/task_types/resmoke_tasks.rs b/src/task_types/resmoke_tasks.rs index 889aa8f..6078545 100644 --- a/src/task_types/resmoke_tasks.rs +++ b/src/task_types/resmoke_tasks.rs @@ -68,6 +68,8 @@ pub struct ResmokeGenParams { pub is_enterprise: bool, /// Arguments to pass to 'run tests' function. pub pass_through_vars: Option>, + /// Name of platform the task will run on. + pub platform: Option, } impl ResmokeGenParams { @@ -200,6 +202,9 @@ pub struct SubSuite { /// Is sub-suite for a enterprise build_variant. pub is_enterprise: bool, + + /// Platform of build_variant the sub-suite is for. + pub platform: Option, } /// Information needed to generate resmoke configuration files for the generated task. @@ -459,6 +464,7 @@ impl GenResmokeTaskServiceImpl { exclude_test_list: None, mv_exclude_tags: multiversion_tags.clone(), is_enterprise: params.is_enterprise, + platform: params.platform.clone(), }); running_tests = vec![]; running_runtime = 0.0; @@ -477,6 +483,7 @@ impl GenResmokeTaskServiceImpl { exclude_test_list: None, mv_exclude_tags: multiversion_tags, is_enterprise: params.is_enterprise, + platform: params.platform.clone(), }); } @@ -558,6 +565,7 @@ impl GenResmokeTaskServiceImpl { exclude_test_list: None, mv_exclude_tags: multiversion_tags.clone(), is_enterprise: params.is_enterprise, + platform: params.platform.clone(), }); current_tests = vec![]; i += 1; @@ -573,6 +581,7 @@ impl GenResmokeTaskServiceImpl { exclude_test_list: None, mv_exclude_tags: multiversion_tags, is_enterprise: params.is_enterprise, + platform: params.platform.clone(), }); } @@ -680,6 +689,7 @@ impl GenResmokeTaskServiceImpl { exclude_test_list: Some(full_test_list), mv_exclude_tags: multiversion_tags, is_enterprise: params.is_enterprise, + platform: params.platform.clone(), }); Ok(sub_suites) @@ -755,6 +765,7 @@ impl GenResmokeTaskService for GenResmokeTaskServiceImpl { sub_suite.index, total_sub_suites, params.is_enterprise, + params.platform.as_deref(), ); let run_test_vars = diff --git a/src/utils/task_name.rs b/src/utils/task_name.rs index 0bf5b60..741319f 100644 --- a/src/utils/task_name.rs +++ b/src/utils/task_name.rs @@ -7,22 +7,28 @@ const GEN_SUFFIX: &str = "_gen"; /// /// # Arguments /// -/// * `parent_name` - Name of task parent task being generated. -/// * `task_index` - Index of sub-task being named. +/// * `display_name` - Name of parent task being generated. +/// * `sub_task_index` - Index of sub-task being named. /// * `total_tasks` - Total number of sub-tasks generated for this parent task. -/// * `variant` - Build Variant being generated. +/// * `is_enterprise` - Whether the task is for an enterprise build variant. +/// * `platform` - Platform that task will run on. pub fn name_generated_task( display_name: &str, sub_task_index: Option, total_tasks: usize, is_enterprise: bool, + platform: Option<&str>, ) -> String { - let suffix = if is_enterprise { + let mut suffix = if is_enterprise { format!("-{}", ENTERPRISE_MODULE) } else { "".to_string() }; + if let Some(platform) = platform { + suffix = format!("-{}{}", platform, suffix) + } + if let Some(index) = sub_task_index { let alignment = (total_tasks as f64).log10().ceil() as usize; format!( @@ -61,22 +67,38 @@ mod tests { use rstest::*; #[rstest] - #[case("task", Some(0), 10, false, "task_0")] - #[case("task", Some(42), 1001, false, "task_0042")] - #[case("task", None, 1001, false, "task_misc")] - #[case("task", None, 0, false, "task_misc")] - #[case("task", Some(0), 10, true, "task_0-enterprise")] - #[case("task", Some(42), 1001, true, "task_0042-enterprise")] - #[case("task", None, 1001, true, "task_misc-enterprise")] - #[case("task", None, 0, true, "task_misc-enterprise")] + #[case("task", Some(0), 10, false, None, "task_0")] + #[case("task", Some(0), 10, false, Some("linux"), "task_0-linux")] + #[case("task", Some(42), 1001, false, None, "task_0042")] + #[case("task", Some(42), 1001, false, Some("linux"), "task_0042-linux")] + #[case("task", None, 1001, false, None, "task_misc")] + #[case("task", None, 1001, false, Some("linux"), "task_misc-linux")] + #[case("task", None, 0, false, None, "task_misc")] + #[case("task", None, 0, false, Some("linux"), "task_misc-linux")] + #[case("task", Some(0), 10, true, None, "task_0-enterprise")] + #[case("task", Some(0), 10, true, Some("linux"), "task_0-linux-enterprise")] + #[case("task", Some(42), 1001, true, None, "task_0042-enterprise")] + #[case( + "task", + Some(42), + 1001, + true, + Some("linux"), + "task_0042-linux-enterprise" + )] + #[case("task", None, 1001, true, None, "task_misc-enterprise")] + #[case("task", None, 1001, true, Some("linux"), "task_misc-linux-enterprise")] + #[case("task", None, 0, true, None, "task_misc-enterprise")] + #[case("task", None, 0, true, Some("linux"), "task_misc-linux-enterprise")] fn test_name_generated_task_should_not_include_suffix( #[case] name: &str, #[case] index: Option, #[case] total: usize, #[case] is_enterprise: bool, + #[case] platform: Option<&str>, #[case] expected: &str, ) { - let task_name = name_generated_task(name, index, total, is_enterprise); + let task_name = name_generated_task(name, index, total, is_enterprise, platform); assert_eq!(task_name, expected); } diff --git a/tests/integration_test.rs b/tests/integration_test.rs index e044ade..697d618 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -28,5 +28,5 @@ fn test_end2end_execution() { assert!(tmp_dir_path.exists()); let files = std::fs::read_dir(tmp_dir_path).unwrap(); - assert_eq!(1561, files.into_iter().collect::>().len()); + assert_eq!(2647, files.into_iter().collect::>().len()); }