From 1856a0358fa5e8ef6a1afe3ed7f87dc8211a9238 Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Mon, 14 Apr 2025 19:14:51 +0000 Subject: [PATCH 01/11] add multiversion task dependency --- src/services/config_extraction.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/services/config_extraction.rs b/src/services/config_extraction.rs index c20d469..3ed5cc7 100644 --- a/src/services/config_extraction.rs +++ b/src/services/config_extraction.rs @@ -127,7 +127,15 @@ impl ConfigExtractionServiceImpl { /// /// List of tasks that should be included as dependencies. fn determine_task_dependencies(&self, task_def: &EvgTask) -> Vec { - let depends_on = self.evg_config_utils.get_task_dependencies(task_def); + let mut depends_on = self.evg_config_utils.get_task_dependencies(task_def); + + if self + .evg_config_utils + .get_task_tags(task_def) + .contains(MULTIVERSION) + { + depends_on.push("multiversion_binary_search".to_string()); + } depends_on .into_iter() From 1d2aaf0e37cc42bbcb66e62597236df2af4b9f56 Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Tue, 15 Apr 2025 12:51:38 +0000 Subject: [PATCH 02/11] add multiversion search task to build variants --- src/lib.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index f5b02d3..13e073a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -730,6 +730,22 @@ impl GenerateTasksService for GenerateTasksServiceImpl { gen_config .gen_task_specs .extend(generated_task.build_task_ref(large_distro)); + + // If any generated task depends on multiversion_binary_search, add multiversion_binary_search to the build variant. + if generated_task.sub_tasks().iter().any(|task| { + match &task.evg_task.depends_on { + Some(deps) => deps + .iter() + .any(|dep| dep.name == "multiversion_binary_search"), + _ => false, + } + }) { + gen_config.gen_task_specs.push(TaskRef { + name: "multiversion_binary_search".to_string(), + distros: None, + activate: Some(false), + }); + } } } From d728597d86b601df86c87ea7105093f3a026419f Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Tue, 15 Apr 2025 20:30:35 +0000 Subject: [PATCH 03/11] testing out using build-variant task dependencies --- Cargo.lock | 5 ++--- Cargo.toml | 3 ++- src/lib.rs | 1 + src/task_types/generated_suite.rs | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0812301..0260f14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -2893,8 +2893,7 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "shrub-rs" version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85ce1399dd2ca316f71b6325f96ed882fe63a1e259862e75571e4100a56a96ea" +source = "git+https://github.com/evergreen-ci/shrub-rs.git?branch=sleaux%2Ftask_deps#1423b85ddcc2f820f8d40793ddd6d3dd86cf9eeb" dependencies = [ "serde", "serde_yaml 0.8.26", diff --git a/Cargo.toml b/Cargo.toml index 8ce0b55..823bb57 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,8 @@ serde = { version = "1.0.206", features = ["derive"] } serde_json = "1.0.124" serde_yaml = "0.9.33" shellexpand = "3.1.0" -shrub-rs = "0.5.5" +#shrub-rs = "0.5.5" +shrub-rs = { git = "https://github.com/evergreen-ci/shrub-rs.git", branch = "sleaux/task_deps" } tokio = { version = "1.39.2", features = ["full"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["json", "fmt", "std"] } diff --git a/src/lib.rs b/src/lib.rs index 13e073a..ce5ffa3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -744,6 +744,7 @@ impl GenerateTasksService for GenerateTasksServiceImpl { name: "multiversion_binary_search".to_string(), distros: None, activate: Some(false), + depends_on: None, }); } } diff --git a/src/task_types/generated_suite.rs b/src/task_types/generated_suite.rs index a9263b1..174f12e 100644 --- a/src/task_types/generated_suite.rs +++ b/src/task_types/generated_suite.rs @@ -59,7 +59,7 @@ pub trait GeneratedSuite: Sync + Send { } sub_task .evg_task - .get_reference(large_distro.map(|d| vec![d]), Some(false)) + .get_reference(sub_task.evg_task.depends_on.clone(), large_distro.map(|d| vec![d]), Some(false)) }) .collect() } From d465d8b9f1af2b511ffdf3bd9b398c52e70d7ac2 Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Thu, 24 Apr 2025 16:18:29 +0000 Subject: [PATCH 04/11] new attempt --- Cargo.lock | 2 +- src/evergreen/evg_config.rs | 1 + src/task_types/generated_suite.rs | 16 ++++++++++++++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 71ff3df..d5e93b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3241,7 +3241,7 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "shrub-rs" version = "0.5.5" -source = "git+https://github.com/evergreen-ci/shrub-rs.git?branch=sleaux%2Ftask_deps#1423b85ddcc2f820f8d40793ddd6d3dd86cf9eeb" +source = "git+https://github.com/evergreen-ci/shrub-rs.git?branch=sleaux%2Ftask_deps#a9b5065fc313dbb161a184dadc0c85595223edff" dependencies = [ "serde", "serde_yaml 0.8.26", diff --git a/src/evergreen/evg_config.rs b/src/evergreen/evg_config.rs index 6f68daa..972bc3d 100644 --- a/src/evergreen/evg_config.rs +++ b/src/evergreen/evg_config.rs @@ -34,6 +34,7 @@ impl EvgProjectConfig { /// * `evg_project_location` - Path to evergreen project configuration to load. pub fn new(evg_project_location: &Path) -> Result { let evg_project = get_project_config(evg_project_location)?; + // dbg!(&evg_project); Ok(Self { evg_project }) } } diff --git a/src/task_types/generated_suite.rs b/src/task_types/generated_suite.rs index 174f12e..c2a7aee 100644 --- a/src/task_types/generated_suite.rs +++ b/src/task_types/generated_suite.rs @@ -57,9 +57,21 @@ pub trait GeneratedSuite: Sync + Send { if sub_task.use_large_distro || sub_task.use_xlarge_distro { large_distro = distro.clone(); } - sub_task + let mut task_ref = sub_task .evg_task - .get_reference(sub_task.evg_task.depends_on.clone(), large_distro.map(|d| vec![d]), Some(false)) + .get_reference(large_distro.map(|d| vec![d]), Some(false)); + + match &sub_task.evg_task.depends_on { + Some(deps) + if deps + .iter() + .any(|dep| dep.name == "multiversion_binary_search") => + { + task_ref.depends_on = sub_task.evg_task.depends_on.clone() + } + _ => (), + } + return task_ref; }) .collect() } From d412d256ec8ad5fa69dab6f5b6cb7cb070b54162 Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Fri, 2 May 2025 15:45:36 +0000 Subject: [PATCH 05/11] upgrade shrub-rs --- Cargo.lock | 5 +++-- Cargo.toml | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5e93b0..b763f8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3240,8 +3240,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "shrub-rs" -version = "0.5.5" -source = "git+https://github.com/evergreen-ci/shrub-rs.git?branch=sleaux%2Ftask_deps#a9b5065fc313dbb161a184dadc0c85595223edff" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a2a925e04e4f04fc528800ffcbb06a21b136226785010e928c26f226656dc57" dependencies = [ "serde", "serde_yaml 0.8.26", diff --git a/Cargo.toml b/Cargo.toml index 4086300..811a64f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,8 +23,7 @@ serde = { version = "1.0.206", features = ["derive"] } serde_json = "1.0.124" serde_yaml = "0.9.33" shellexpand = "3.1.0" -#shrub-rs = "0.5.5" -shrub-rs = { git = "https://github.com/evergreen-ci/shrub-rs.git", branch = "sleaux/task_deps" } +shrub-rs = "0.6.0" tokio = { version = "1.39.2", features = ["full"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["json", "fmt", "std"] } From 7677ec99736d1daeed632d861666fa4f994b2630 Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Tue, 6 May 2025 19:50:33 +0000 Subject: [PATCH 06/11] fix variant depends_on override --- src/evergreen/evg_config_utils.rs | 21 ++++++++++++++++ src/lib.rs | 41 +++++++++++++++++++++++++++++-- src/task_types/burn_in_tests.rs | 10 +++++++- src/task_types/fuzzer_tasks.rs | 2 +- src/task_types/generated_suite.rs | 22 +++++++---------- src/task_types/resmoke_tasks.rs | 2 +- 6 files changed, 80 insertions(+), 18 deletions(-) diff --git a/src/evergreen/evg_config_utils.rs b/src/evergreen/evg_config_utils.rs index 968ad72..298a4ef 100644 --- a/src/evergreen/evg_config_utils.rs +++ b/src/evergreen/evg_config_utils.rs @@ -6,6 +6,7 @@ use lazy_static::lazy_static; use regex::Regex; use shrub_rs::models::commands::EvgCommand::Function; use shrub_rs::models::params::ParamValue; +use shrub_rs::models::task::TaskDependency; use shrub_rs::models::{commands::FunctionCall, task::EvgTask, variant::BuildVariant}; use crate::evergreen_names::{ @@ -101,6 +102,12 @@ pub trait EvgConfigUtils: Sync + Send { /// List of task names the task depends on. fn get_task_dependencies(&self, task: &EvgTask) -> Vec; + fn get_task_ref_dependencies( + &self, + task_name: &str, + build_variant: &BuildVariant, + ) -> Option>; + /// Lookup the given variable in the vars section of the 'generate resmoke task' func. /// /// # Arguments @@ -430,6 +437,20 @@ impl EvgConfigUtils for EvgConfigUtilsImpl { dependencies.unwrap_or_default() } + fn get_task_ref_dependencies( + &self, + task_name: &str, + build_variant: &BuildVariant, + ) -> Option> { + for task_ref in &build_variant.tasks { + if task_ref.name == task_name { + let dependencies = task_ref.depends_on.clone(); + return dependencies; + } + } + None + } + /// Lookup the given variable in the vars section of the 'generate resmoke task' func. /// /// # Arguments diff --git a/src/lib.rs b/src/lib.rs index 2de8f7d..3ccea70 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,7 +35,7 @@ use resmoke::{ use services::config_extraction::{ConfigExtractionService, ConfigExtractionServiceImpl}; use shrub_rs::models::{ project::EvgProject, - task::{EvgTask, TaskRef}, + task::{EvgTask, TaskDependency, TaskRef}, variant::{BuildVariant, DisplayTask}, }; use task_types::{ @@ -748,9 +748,37 @@ impl GenerateTasksService for GenerateTasksServiceImpl { gen_config .display_tasks .push(generated_task.build_display_task()); + + let mut task_ref_depends_on = None; + if generated_task.sub_tasks().iter().any(|task| { + match &task.evg_task.depends_on { + Some(deps) => deps + .iter() + .any(|dep| dep.name == "multiversion_binary_search"), + _ => true, + } + }) { + let base_dependencies = self + .evg_config_utils + .get_task_ref_dependencies(&task.name, build_variant); + + if let Some(deps) = base_dependencies { + task_ref_depends_on = Some( + [ + deps.clone(), + vec![TaskDependency { + name: "multiversion_binary_search".to_string(), + variant: None, + }], + ] + .concat(), + ); + } + } + gen_config .gen_task_specs - .extend(generated_task.build_task_ref(large_distro)); + .extend(generated_task.build_task_ref(large_distro, task_ref_depends_on)); // If any generated task depends on multiversion_binary_search, add multiversion_binary_search to the build variant. if generated_task.sub_tasks().iter().any(|task| { @@ -1033,6 +1061,7 @@ pub async fn build_s3_client() -> aws_sdk_s3::Client { #[cfg(test)] mod tests { use rstest::rstest; + use shrub_rs::models::task::TaskDependency; use crate::{ evergreen::evg_config_utils::MultiversionGenerateTaskConfig, @@ -1160,6 +1189,14 @@ mod tests { todo!() } + fn get_task_ref_dependencies( + &self, + _task_name: &str, + _build_variant: &BuildVariant, + ) -> Option> { + todo!() + } + fn get_gen_task_var<'a>(&self, _task: &'a EvgTask, _var: &str) -> Option<&'a str> { todo!() } diff --git a/src/task_types/burn_in_tests.rs b/src/task_types/burn_in_tests.rs index f978009..8f7e4ab 100644 --- a/src/task_types/burn_in_tests.rs +++ b/src/task_types/burn_in_tests.rs @@ -430,7 +430,7 @@ impl BurnInService for BurnInServiceImpl { gen_config .gen_task_specs - .extend(generated_task.build_task_ref(large_distro)); + .extend(generated_task.build_task_ref(large_distro, None)); gen_config .display_tasks .push(generated_task.build_display_task()); @@ -730,6 +730,14 @@ mod tests { todo!() } + fn get_task_ref_dependencies( + &self, + _task_name: &str, + _build_variant: &BuildVariant, + ) -> Option> { + todo!() + } + fn get_gen_task_var<'a>(&self, _task: &'a EvgTask, _var: &str) -> Option<&'a str> { todo!() } diff --git a/src/task_types/fuzzer_tasks.rs b/src/task_types/fuzzer_tasks.rs index 59e1823..d3329ee 100644 --- a/src/task_types/fuzzer_tasks.rs +++ b/src/task_types/fuzzer_tasks.rs @@ -439,7 +439,7 @@ mod tests { ], }; - let task_refs = fuzzer_task.build_task_ref(Some("distro".to_string())); + let task_refs = fuzzer_task.build_task_ref(Some("distro".to_string()), None); for task in task_refs { assert_eq!(task.distros.as_ref(), None); diff --git a/src/task_types/generated_suite.rs b/src/task_types/generated_suite.rs index c2a7aee..dfbf95c 100644 --- a/src/task_types/generated_suite.rs +++ b/src/task_types/generated_suite.rs @@ -1,5 +1,5 @@ use shrub_rs::models::{ - task::{EvgTask, TaskRef}, + task::{EvgTask, TaskDependency, TaskRef}, variant::DisplayTask, }; @@ -49,7 +49,11 @@ pub trait GeneratedSuite: Sync + Send { } /// Build a shrub task reference for this generated task. - fn build_task_ref(&self, distro: Option) -> Vec { + fn build_task_ref( + &self, + distro: Option, + depends_on: Option>, + ) -> Vec { self.sub_tasks() .iter() .map(|sub_task| { @@ -61,17 +65,9 @@ pub trait GeneratedSuite: Sync + Send { .evg_task .get_reference(large_distro.map(|d| vec![d]), Some(false)); - match &sub_task.evg_task.depends_on { - Some(deps) - if deps - .iter() - .any(|dep| dep.name == "multiversion_binary_search") => - { - task_ref.depends_on = sub_task.evg_task.depends_on.clone() - } - _ => (), - } - return task_ref; + task_ref.depends_on = depends_on.clone(); + + task_ref }) .collect() } diff --git a/src/task_types/resmoke_tasks.rs b/src/task_types/resmoke_tasks.rs index 2a99c39..387bc98 100644 --- a/src/task_types/resmoke_tasks.rs +++ b/src/task_types/resmoke_tasks.rs @@ -1053,7 +1053,7 @@ mod tests { .collect(), }; - let task_refs = gen_suite.build_task_ref(Some(distro.clone())); + let task_refs = gen_suite.build_task_ref(Some(distro.clone()), None); for (i, task) in task_refs.iter().enumerate() { assert_eq!(task.name, format!("sub_suite_name_{}", i)); From 65b75f57697e1851dbc47534664b50d36bf31062 Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Wed, 7 May 2025 15:38:09 +0000 Subject: [PATCH 07/11] some cleanup and tests --- src/evergreen/evg_config.rs | 1 - src/evergreen_names.rs | 2 + src/lib.rs | 64 +++++++++++++------------------ src/services/config_extraction.rs | 19 +++++---- src/task_types/generated_suite.rs | 13 +++++++ tests/data/evergreen.yml | 3 ++ 6 files changed, 55 insertions(+), 47 deletions(-) diff --git a/src/evergreen/evg_config.rs b/src/evergreen/evg_config.rs index 972bc3d..6f68daa 100644 --- a/src/evergreen/evg_config.rs +++ b/src/evergreen/evg_config.rs @@ -34,7 +34,6 @@ impl EvgProjectConfig { /// * `evg_project_location` - Path to evergreen project configuration to load. pub fn new(evg_project_location: &Path) -> Result { let evg_project = get_project_config(evg_project_location)?; - // dbg!(&evg_project); Ok(Self { evg_project }) } } diff --git a/src/evergreen_names.rs b/src/evergreen_names.rs index b77d07b..830bb3f 100644 --- a/src/evergreen_names.rs +++ b/src/evergreen_names.rs @@ -44,6 +44,8 @@ pub const BURN_IN_TESTS: &str = "burn_in_tests_gen"; pub const BURN_IN_TAGS: &str = "burn_in_tags_gen"; /// Name of burn_in_tasks task. pub const BURN_IN_TASKS: &str = "burn_in_tasks_gen"; +/// Name of multiversion binary selection task. +pub const MULTIVERSION_BINARY_SELECTION: &str = "multiversion_binary_selection"; // Vars /// Variable that indicates a task is a fuzzer. diff --git a/src/lib.rs b/src/lib.rs index 3ccea70..1fd14fa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,7 +25,8 @@ use evergreen::{ }; use evergreen_names::{ BURN_IN_TAGS, BURN_IN_TAG_COMPILE_TASK_DEPENDENCY, BURN_IN_TAG_INCLUDE_BUILD_VARIANTS, - BURN_IN_TASKS, BURN_IN_TESTS, ENTERPRISE_MODULE, GENERATOR_TASKS, UNIQUE_GEN_SUFFIX_EXPANSION, + BURN_IN_TASKS, BURN_IN_TESTS, ENTERPRISE_MODULE, GENERATOR_TASKS, + MULTIVERSION_BINARY_SELECTION, UNIQUE_GEN_SUFFIX_EXPANSION, }; use generate_sub_tasks_config::GenerateSubTasksConfig; use resmoke::{ @@ -749,48 +750,35 @@ impl GenerateTasksService for GenerateTasksServiceImpl { .display_tasks .push(generated_task.build_display_task()); - let mut task_ref_depends_on = None; - if generated_task.sub_tasks().iter().any(|task| { - match &task.evg_task.depends_on { - Some(deps) => deps - .iter() - .any(|dep| dep.name == "multiversion_binary_search"), - _ => true, - } - }) { - let base_dependencies = self - .evg_config_utils - .get_task_ref_dependencies(&task.name, build_variant); - - if let Some(deps) = base_dependencies { - task_ref_depends_on = Some( - [ - deps.clone(), - vec![TaskDependency { - name: "multiversion_binary_search".to_string(), - variant: None, - }], - ] - .concat(), - ); - } + // If a task is a multiversion task and it has dependencies overriden on the task + // reference of the build variant, add the MULTIVERSION_BINARY_SELECTION task to + // those overrides too. + let mut task_ref_dependencies = self + .evg_config_utils + .get_task_ref_dependencies(&task.name, build_variant); + + if generated_task.is_multiversion() && task_ref_dependencies.is_some() { + task_ref_dependencies = Some( + [ + task_ref_dependencies.unwrap(), + vec![TaskDependency { + name: MULTIVERSION_BINARY_SELECTION.to_string(), + variant: None, + }], + ] + .concat(), + ); } gen_config .gen_task_specs - .extend(generated_task.build_task_ref(large_distro, task_ref_depends_on)); - - // If any generated task depends on multiversion_binary_search, add multiversion_binary_search to the build variant. - if generated_task.sub_tasks().iter().any(|task| { - match &task.evg_task.depends_on { - Some(deps) => deps - .iter() - .any(|dep| dep.name == "multiversion_binary_search"), - _ => false, - } - }) { + .extend(generated_task.build_task_ref(large_distro, task_ref_dependencies)); + + // If any generated task is multiversion, ensure the + // MULTIVERSION_BINARY_SELECTION task is added to the build variant. + if generated_task.is_multiversion() { gen_config.gen_task_specs.push(TaskRef { - name: "multiversion_binary_search".to_string(), + name: MULTIVERSION_BINARY_SELECTION.to_string(), distros: None, activate: Some(false), depends_on: None, diff --git a/src/services/config_extraction.rs b/src/services/config_extraction.rs index 2a0bd93..a5a7f96 100644 --- a/src/services/config_extraction.rs +++ b/src/services/config_extraction.rs @@ -7,10 +7,10 @@ use crate::{ evergreen::evg_config_utils::EvgConfigUtils, evergreen_names::{ CONTINUE_ON_FAILURE, FUZZER_PARAMETERS, IDLE_TIMEOUT, LARGE_DISTRO_EXPANSION, - LAST_VERSIONS_EXPANSION, MULTIVERSION, NO_MULTIVERSION_GENERATE_TASKS, NPM_COMMAND, - NUM_FUZZER_FILES, NUM_FUZZER_TASKS, REPEAT_SUITES, RESMOKE_ARGS, RESMOKE_JOBS_MAX, - SHOULD_SHUFFLE_TESTS, UNIQUE_GEN_SUFFIX_EXPANSION, USE_LARGE_DISTRO, USE_XLARGE_DISTRO, - XLARGE_DISTRO_EXPANSION, + LAST_VERSIONS_EXPANSION, MULTIVERSION, MULTIVERSION_BINARY_SELECTION, + NO_MULTIVERSION_GENERATE_TASKS, NPM_COMMAND, NUM_FUZZER_FILES, NUM_FUZZER_TASKS, + REPEAT_SUITES, RESMOKE_ARGS, RESMOKE_JOBS_MAX, SHOULD_SHUFFLE_TESTS, + UNIQUE_GEN_SUFFIX_EXPANSION, USE_LARGE_DISTRO, USE_XLARGE_DISTRO, XLARGE_DISTRO_EXPANSION, }, generate_sub_tasks_config::GenerateSubTasksConfig, task_types::{ @@ -134,7 +134,7 @@ impl ConfigExtractionServiceImpl { .get_task_tags(task_def) .contains(MULTIVERSION) { - depends_on.push("multiversion_binary_search".to_string()); + depends_on.push(MULTIVERSION_BINARY_SELECTION.to_string()); } depends_on @@ -407,12 +407,14 @@ mod tests { // Tests for determine_task_dependencies. #[rstest] #[case( - vec![], vec![] + vec![], vec![], vec![] )] - #[case(vec!["dependency_0", "dependency_1"], vec!["dependency_0", "dependency_1"])] - #[case(vec!["dependency_0", "generating_task"], vec!["dependency_0"])] + #[case(vec!["dependency_0", "dependency_1"], vec![], vec!["dependency_0", "dependency_1"])] + #[case(vec!["dependency_0", "generating_task"], vec![], vec!["dependency_0"])] + #[case(vec!["dependency_0", "generating_task"], vec!["multiversion".to_string()], vec!["dependency_0", MULTIVERSION_BINARY_SELECTION])] fn test_determine_task_dependencies( #[case] depends_on: Vec<&str>, + #[case] tags: Vec, #[case] expected_deps: Vec<&str>, ) { let config_extraction_service = build_mocked_config_extraction_service(); @@ -426,6 +428,7 @@ mod tests { }) .collect(), ), + tags: Some(tags), ..Default::default() }; diff --git a/src/task_types/generated_suite.rs b/src/task_types/generated_suite.rs index dfbf95c..ba6f9eb 100644 --- a/src/task_types/generated_suite.rs +++ b/src/task_types/generated_suite.rs @@ -3,6 +3,8 @@ use shrub_rs::models::{ variant::DisplayTask, }; +use crate::evergreen_names::MULTIVERSION_BINARY_SELECTION; + /// Definition of a generated sub task. #[derive(Clone, Debug, Default)] pub struct GeneratedSubTask { @@ -48,6 +50,17 @@ pub trait GeneratedSuite: Sync + Send { } } + fn is_multiversion(&self) -> bool { + self.sub_tasks() + .iter() + .any(|task| match &task.evg_task.depends_on { + Some(deps) => deps + .iter() + .any(|dep| dep.name == MULTIVERSION_BINARY_SELECTION), + _ => false, + }) + } + /// Build a shrub task reference for this generated task. fn build_task_ref( &self, diff --git a/tests/data/evergreen.yml b/tests/data/evergreen.yml index d5995a7..438d70f 100644 --- a/tests/data/evergreen.yml +++ b/tests/data/evergreen.yml @@ -7942,6 +7942,9 @@ buildvariants: test_flags: >- --runAllFeatureFlagTests --excludeWithAnyTags=incompatible_with_shard_merge + depends_on: + - name: archive_dist_test + variant: enterprise-rhel-80-64-bit-dynamic-required tasks: &enterprise-rhel-80-64-bit-dynamic-all-feature-flags-tasks - name: change_streams_per_shard_cursor_passthrough - name: cqf From b42e3121de5a6e28b51be9ddefdd639e5f5e8688 Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Wed, 7 May 2025 15:43:30 +0000 Subject: [PATCH 08/11] rename task --- src/evergreen_names.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evergreen_names.rs b/src/evergreen_names.rs index 830bb3f..5520cf3 100644 --- a/src/evergreen_names.rs +++ b/src/evergreen_names.rs @@ -45,7 +45,7 @@ pub const BURN_IN_TAGS: &str = "burn_in_tags_gen"; /// Name of burn_in_tasks task. pub const BURN_IN_TASKS: &str = "burn_in_tasks_gen"; /// Name of multiversion binary selection task. -pub const MULTIVERSION_BINARY_SELECTION: &str = "multiversion_binary_selection"; +pub const MULTIVERSION_BINARY_SELECTION: &str = "select_multiversion_binaries"; // Vars /// Variable that indicates a task is a fuzzer. From 8aa6bbdc569e41fed863d0f51709d85cb07ce5de Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Wed, 7 May 2025 17:05:41 +0000 Subject: [PATCH 09/11] bump version --- CHANGELOG.md | 3 +++ Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d20372..02c7cc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 3.0.0 - 2025-05-07 +* Generate multiversion binary selection tasks + ## 2.0.0 - 2025-04-23 * Allow splitting large tasks on required variants based on total test runtime * Use the AWS-sdk for accessing the s3 bucket of test statistics diff --git a/Cargo.lock b/Cargo.lock index b763f8f..070b11b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2070,7 +2070,7 @@ dependencies = [ [[package]] name = "mongo-task-generator" -version = "2.0.0" +version = "3.0.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 811a64f..2dabdab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "mongo-task-generator" description = "Dynamically split evergreen tasks into subtasks for testing the 10gen/mongo project." license = "Apache-2.0" -version = "2.0.0" +version = "3.0.0" repository = "https://github.com/mongodb/mongo-task-generator" authors = ["DevProd Correctness Team "] edition = "2018" From 7f1163eb4e3677a6576981aa7f7430d3f2b9cae7 Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Wed, 7 May 2025 17:27:20 +0000 Subject: [PATCH 10/11] only append task once per variant --- src/lib.rs | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1fd14fa..6ec062d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -709,6 +709,7 @@ impl GenerateTasksService for GenerateTasksServiceImpl { .infer_build_variant_platform(build_variant); let mut gen_config = GeneratedConfig::new(); let mut generating_tasks = vec![]; + let mut includes_multiversion_tasks = false; for task in &build_variant.tasks { if task.name == BURN_IN_TAGS { if self.gen_burn_in { @@ -770,23 +771,27 @@ impl GenerateTasksService for GenerateTasksServiceImpl { ); } + if generated_task.is_multiversion() { + includes_multiversion_tasks = true; + } + gen_config .gen_task_specs .extend(generated_task.build_task_ref(large_distro, task_ref_dependencies)); - - // If any generated task is multiversion, ensure the - // MULTIVERSION_BINARY_SELECTION task is added to the build variant. - if generated_task.is_multiversion() { - gen_config.gen_task_specs.push(TaskRef { - name: MULTIVERSION_BINARY_SELECTION.to_string(), - distros: None, - activate: Some(false), - depends_on: None, - }); - } } } + // If any generated task is multiversion, ensure the + // MULTIVERSION_BINARY_SELECTION task is added to the build variant. + if includes_multiversion_tasks { + gen_config.gen_task_specs.push(TaskRef { + name: MULTIVERSION_BINARY_SELECTION.to_string(), + distros: None, + activate: Some(false), + depends_on: None, + }); + } + if !generating_tasks.is_empty() { // Put all the "_gen" tasks into a display task to hide them from view. gen_config.display_tasks.push(DisplayTask { From 4b78b05191bfe29b689a663f6ae4c38d20136036 Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Wed, 7 May 2025 18:42:35 +0000 Subject: [PATCH 11/11] avoid redundantly specifying dependencies --- src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 6ec062d..e7396ce 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -769,6 +769,8 @@ impl GenerateTasksService for GenerateTasksServiceImpl { ] .concat(), ); + } else { + task_ref_dependencies = None; } if generated_task.is_multiversion() {