From 1cb7adc044d7a2874e866b1b207f9552204c4aba Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Thu, 17 Apr 2025 14:47:33 +0000 Subject: [PATCH 1/3] Make number of sub-tasks configurable --- Cargo.lock | 2 +- Cargo.toml | 3 +- docs/generating_tasks.md | 3 +- src/lib.rs | 9 +-- src/services/config_extraction.rs | 9 +++ src/task_types/resmoke_tasks.rs | 125 ++++++++++++++++++------------ tests/data/evergreen.yml | 3 + tests/integration_test.rs | 2 +- 8 files changed, 96 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b46c840..da602ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1635,7 +1635,7 @@ dependencies = [ [[package]] name = "mongo-task-generator" -version = "1.0.0" +version = "1.1.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 940673a..9d22aba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,10 +2,11 @@ name = "mongo-task-generator" description = "Dynamically split evergreen tasks into subtasks for testing the 10gen/mongo project." license = "Apache-2.0" -version = "1.0.0" +version = "1.1.0" repository = "https://github.com/mongodb/mongo-task-generator" authors = ["DevProd Correctness Team "] edition = "2018" +rust-version = "1.75" [dependencies] anyhow = "1.0.86" diff --git a/docs/generating_tasks.md b/docs/generating_tasks.md index 8941881..b20b46e 100644 --- a/docs/generating_tasks.md +++ b/docs/generating_tasks.md @@ -99,7 +99,7 @@ Looking at a [sample resmoke-based](https://github.com/mongodb/mongo/blob/852c5d ``` Like fuzzer tasks, task generation is indicated by including the `"generate resmoke tasks"` function. -Additionally, the 3 parameters here will impact how the task is generated. +Additionally, the 4 parameters here will impact how the task is generated. * **suite**: By default, the name of the task (with the `_gen` suffix stripped off) will be used to determine which resmoke suite to base the generated sub-tasks on. This can be overwritten with @@ -112,6 +112,7 @@ Additionally, the 3 parameters here will impact how the task is generated. variable is set to `"true"`, certain tasks will use an even larger distro that can be defined with the `xlarge_distro_name` expansion in the build variant. When the `xlarge_distro_name` expansion is not defined, it will fallback to the defined `large_distro_name` expansion in the build variant +* **num_tasks**: The number of generated sub-tasks to split into. (Default 5). **Note**: If a task has the `use_large_distro` value defined, but is added to a build variant without a `large_distro_name`, it will trigger a failure. This can be supported by using the diff --git a/src/lib.rs b/src/lib.rs index f5b02d3..d72c65e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -58,7 +58,7 @@ mod utils; const BURN_IN_TESTS_PREFIX: &str = "burn_in_tests"; const BURN_IN_TASKS_PREFIX: &str = "burn_in_tasks"; const BURN_IN_BV_SUFFIX: &str = "generated-by-burn-in-tags"; -const MAX_SUB_TASKS_PER_TASK: usize = 5; +const DEFAULT_SUB_TASKS_PER_TASK: usize = 5; type GenTaskCollection = HashMap>; @@ -206,11 +206,8 @@ impl Dependencies { 32, ))); let enterprise_dir = evg_config_service.get_module_dir(ENTERPRISE_MODULE); - let gen_resmoke_config = GenResmokeConfig::new( - MAX_SUB_TASKS_PER_TASK, - execution_config.use_task_split_fallback, - enterprise_dir, - ); + let gen_resmoke_config = + GenResmokeConfig::new(execution_config.use_task_split_fallback, enterprise_dir); let gen_resmoke_task_service = Arc::new(GenResmokeTaskServiceImpl::new( task_history_service, discovery_service, diff --git a/src/services/config_extraction.rs b/src/services/config_extraction.rs index c20d469..c0a9cc3 100644 --- a/src/services/config_extraction.rs +++ b/src/services/config_extraction.rs @@ -18,6 +18,7 @@ use crate::{ multiversion::MultiversionService, resmoke_tasks::ResmokeGenParams, }, utils::task_name::remove_gen_suffix, + DEFAULT_SUB_TASKS_PER_TASK, }; /// Interface for performing extractions of evergreen project configuration. @@ -248,6 +249,13 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl { .evg_config_utils .lookup_build_variant_expansion(UNIQUE_GEN_SUFFIX_EXPANSION, variant); } + let num_tasks = match self + .evg_config_utils + .get_gen_task_var(task_def, "num_tasks") + { + Some(str) => str.parse().unwrap(), + _ => DEFAULT_SUB_TASKS_PER_TASK, + }; Ok(ResmokeGenParams { task_name, @@ -289,6 +297,7 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl { pass_through_vars: self.evg_config_utils.get_gen_task_vars(task_def), platform, gen_task_suffix, + num_tasks, }) } diff --git a/src/task_types/resmoke_tasks.rs b/src/task_types/resmoke_tasks.rs index 97cf59e..f771ed5 100644 --- a/src/task_types/resmoke_tasks.rs +++ b/src/task_types/resmoke_tasks.rs @@ -33,6 +33,7 @@ use crate::{ }, resmoke::resmoke_proxy::TestDiscovery, utils::{fs_service::FsService, task_name::name_generated_task}, + DEFAULT_SUB_TASKS_PER_TASK, }; use super::{ @@ -42,7 +43,7 @@ use super::{ }; /// Parameters describing how a specific resmoke suite should be generated. -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug)] pub struct ResmokeGenParams { /// Name of task being generated. pub task_name: String, @@ -76,6 +77,32 @@ pub struct ResmokeGenParams { pub platform: Option, /// Name of variant specific suffix to add to tasks pub gen_task_suffix: Option, + /// Number of sub-tasks fuzzer should generate. + pub num_tasks: usize, +} + +impl Default for ResmokeGenParams { + fn default() -> ResmokeGenParams { + ResmokeGenParams { + num_tasks: DEFAULT_SUB_TASKS_PER_TASK, + task_name: Default::default(), + multiversion_generate_tasks: Default::default(), + suite_name: Default::default(), + use_large_distro: Default::default(), + use_xlarge_distro: Default::default(), + require_multiversion_setup: Default::default(), + require_multiversion_generate_tasks: Default::default(), + repeat_suites: Default::default(), + resmoke_args: Default::default(), + resmoke_jobs_max: Default::default(), + config_location: Default::default(), + dependencies: Default::default(), + is_enterprise: Default::default(), + pass_through_vars: Default::default(), + platform: Default::default(), + gen_task_suffix: Default::default(), + } + } } impl ResmokeGenParams { @@ -291,9 +318,6 @@ pub trait GenResmokeTaskService: Sync + Send { #[derive(Debug, Clone)] pub struct GenResmokeConfig { - /// Max number of suites to split tasks into. - n_suites: usize, - /// Disable evergreen task-history queries and use task splitting fallback. use_task_split_fallback: bool, @@ -314,13 +338,8 @@ impl GenResmokeConfig { /// # Returns /// /// New instance of `GenResmokeConfig`. - pub fn new( - n_suites: usize, - use_task_split_fallback: bool, - enterprise_dir: Option, - ) -> Self { + pub fn new(use_task_split_fallback: bool, enterprise_dir: Option) -> Self { Self { - n_suites, use_task_split_fallback, enterprise_dir, } @@ -408,7 +427,7 @@ impl GenResmokeTaskServiceImpl { .iter() .fold(0.0, |init, (_, item)| init + item.average_runtime); - let max_tasks = min(self.config.n_suites, test_list.len()); + let max_tasks = min(params.num_tasks, test_list.len()); let runtime_per_subtask = total_runtime / max_tasks as f64; event!( Level::INFO, @@ -517,7 +536,7 @@ impl GenResmokeTaskServiceImpl { return Ok(sub_suites); } - let n = min(test_list.len(), self.config.n_suites); + let n = min(test_list.len(), params.num_tasks); let len = test_list.len(); let (quo, rem) = (len / n, len % n); let split = (quo + 1) * rem; @@ -1106,7 +1125,6 @@ mod tests { fn build_mocked_service( test_list: Vec, task_history: TaskRuntimeHistory, - n_suites: usize, ) -> GenResmokeTaskServiceImpl { let test_discovery = MockTestDiscovery { test_list }; let multiversion_service = MockMultiversionService {}; @@ -1116,7 +1134,7 @@ mod tests { let fs_service = MockFsService {}; let resmoke_config_actor = MockResmokeConfigActor {}; - let config = GenResmokeConfig::new(n_suites, false, Some(MOCK_ENTERPRISE_DIR.to_string())); + let config = GenResmokeConfig::new(false, Some(MOCK_ENTERPRISE_DIR.to_string())); GenResmokeTaskServiceImpl::new( Arc::new(task_history_service), @@ -1141,7 +1159,7 @@ mod tests { // In this test we will create 3 subtasks with 6 tests. The first sub task should contain // a single test. The second 2 tests and the third 3 tests. We will set the test runtimes // to make this happen. - let n_suites = 3; + let num_tasks = 3; let test_list: Vec = (0..6) .into_iter() .map(|i| format!("test_{}.js", i)) @@ -1157,10 +1175,10 @@ mod tests { "test_5".to_string() => build_mock_test_runtime("test_5.js", 30.0), }, }; - let gen_resmoke_service = - build_mocked_service(test_list.clone(), task_history.clone(), n_suites); + let gen_resmoke_service = build_mocked_service(test_list.clone(), task_history.clone()); let params = ResmokeGenParams { + num_tasks, ..Default::default() }; @@ -1168,7 +1186,7 @@ mod tests { .split_task(¶ms, &task_history, None, None) .unwrap(); - assert_eq!(sub_suites.len(), n_suites); + assert_eq!(sub_suites.len(), num_tasks); let suite_0 = &sub_suites[0]; assert!(suite_0.test_list.contains(&"test_0.js".to_string())); let suite_1 = &sub_suites[1]; @@ -1182,7 +1200,7 @@ mod tests { #[test] fn test_split_task_with_missing_history_should_split_tasks_equally() { - let n_suites = 3; + let num_tasks = 3; let test_list: Vec = (0..12) .into_iter() .map(|i| format!("test_{}.js", i)) @@ -1195,9 +1213,10 @@ mod tests { "test_2".to_string() => build_mock_test_runtime("test_2.js", 50.0), }, }; - let gen_resmoke_service = build_mocked_service(test_list, task_history.clone(), n_suites); + let gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); let params = ResmokeGenParams { + num_tasks, ..Default::default() }; @@ -1205,7 +1224,7 @@ mod tests { .split_task(¶ms, &task_history, None, None) .unwrap(); - assert_eq!(sub_suites.len(), n_suites); + assert_eq!(sub_suites.len(), num_tasks); let suite_0 = &sub_suites[0]; assert_eq!(suite_0.test_list.len(), 4); let suite_1 = &sub_suites[1]; @@ -1216,7 +1235,7 @@ mod tests { #[test] fn test_split_tasks_should_include_multiversion_information() { - let n_suites = 3; + let num_tasks = 3; let test_list: Vec = (0..3) .into_iter() .map(|i| format!("test_{}.js", i)) @@ -1229,9 +1248,10 @@ mod tests { "test_2".to_string() => build_mock_test_runtime("test_2.js", 50.0), }, }; - let gen_resmoke_service = build_mocked_service(test_list, task_history.clone(), n_suites); + let gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); let params = ResmokeGenParams { + num_tasks, ..Default::default() }; @@ -1244,7 +1264,7 @@ mod tests { ) .unwrap(); - assert_eq!(sub_suites.len(), n_suites); + assert_eq!(sub_suites.len(), num_tasks); for sub_suite in sub_suites { assert_eq!(sub_suite.name, "multiversion_test"); assert_eq!( @@ -1258,7 +1278,7 @@ mod tests { #[test] fn test_split_task_fallback_should_split_tasks_count() { - let n_suites = 3; + let num_tasks = 3; let n_tests = 6; let test_list: Vec = (0..n_tests) .into_iter() @@ -1268,10 +1288,10 @@ mod tests { task_name: "my task".to_string(), test_map: hashmap! {}, }; - let gen_resmoke_service = - build_mocked_service(test_list.clone(), task_history.clone(), n_suites); + let gen_resmoke_service = build_mocked_service(test_list.clone(), task_history.clone()); let params = ResmokeGenParams { + num_tasks, ..Default::default() }; @@ -1279,9 +1299,9 @@ mod tests { .split_task_fallback(¶ms, None, None) .unwrap(); - assert_eq!(sub_suites.len(), n_suites); + assert_eq!(sub_suites.len(), num_tasks); for sub_suite in &sub_suites { - assert_eq!(sub_suite.test_list.len(), n_tests / n_suites); + assert_eq!(sub_suite.test_list.len(), n_tests / num_tasks); } let all_tests: Vec = sub_suites @@ -1296,7 +1316,7 @@ mod tests { #[test] fn test_split_task_fallback_has_remainder() { - let n_suites = 3; + let num_tasks = 3; let n_tests = 4; let test_list: Vec = (0..n_tests) .into_iter() @@ -1306,10 +1326,10 @@ mod tests { task_name: "my task".to_string(), test_map: hashmap! {}, }; - let gen_resmoke_service = - build_mocked_service(test_list.clone(), task_history.clone(), n_suites); + let gen_resmoke_service = build_mocked_service(test_list.clone(), task_history.clone()); let params = ResmokeGenParams { + num_tasks, ..Default::default() }; @@ -1317,7 +1337,7 @@ mod tests { .split_task_fallback(¶ms, None, None) .unwrap(); - assert_eq!(sub_suites.len(), n_suites); + assert_eq!(sub_suites.len(), num_tasks); let all_tests: Vec = sub_suites .iter() .flat_map(|s| s.test_list.clone()) @@ -1330,16 +1350,16 @@ mod tests { #[test] fn test_split_task_fallback_empty_suite() { - let n_suites = 1; + let num_tasks = 1; let test_list = vec![]; let task_history = TaskRuntimeHistory { task_name: "my task".to_string(), test_map: hashmap! {}, }; - let gen_resmoke_service = - build_mocked_service(test_list.clone(), task_history.clone(), n_suites); + let gen_resmoke_service = build_mocked_service(test_list.clone(), task_history.clone()); let params = ResmokeGenParams { + num_tasks, ..Default::default() }; @@ -1358,7 +1378,7 @@ mod tests { #[case] is_enterprise: bool, #[case] expected_tests: usize, ) { - let n_suites = 3; + let num_tasks = 3; let mut test_list: Vec = (0..6) .into_iter() .map(|i| format!("test_{}.js", i)) @@ -1373,10 +1393,11 @@ mod tests { task_name: "my task".to_string(), test_map: hashmap! {}, }; - let gen_resmoke_service = build_mocked_service(test_list, task_history.clone(), n_suites); + let gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); let params = ResmokeGenParams { is_enterprise, + num_tasks, ..Default::default() }; @@ -1397,7 +1418,7 @@ mod tests { #[case] is_enterprise: bool, #[case] expected_tests: usize, ) { - let n_suites = 3; + let num_tasks = 3; let mut test_list: Vec = (0..6) .into_iter() .map(|i| format!("test_{}.js", i)) @@ -1412,12 +1433,12 @@ mod tests { task_name: "my task".to_string(), test_map: hashmap! {}, }; - let mut gen_resmoke_service = - build_mocked_service(test_list, task_history.clone(), n_suites); + let mut gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); gen_resmoke_service.config.enterprise_dir = None; let params = ResmokeGenParams { is_enterprise, + num_tasks, ..Default::default() }; @@ -1445,6 +1466,7 @@ mod tests { old_version: "last-continuous".to_string(), }, ]), + num_tasks: 1, ..Default::default() }; let task_history = TaskRuntimeHistory { @@ -1457,7 +1479,7 @@ mod tests { "test_2.js".to_string(), "test_3.js".to_string(), ]; - let gen_resmoke_service = build_mocked_service(test_list.clone(), task_history, 1); + let gen_resmoke_service = build_mocked_service(test_list.clone(), task_history); let suite_list = gen_resmoke_service .create_multiversion_tasks(¶ms, "build_variant") @@ -1487,7 +1509,7 @@ mod tests { // In this test we will create 3 subtasks with 6 tests. The first sub task should contain // a single test. The second 2 tests and the third 3 tests. We will set the test runtimes // to make this happen. - let n_suites = 3; + let num_tasks = 3; let test_list: Vec = (0..6) .into_iter() .map(|i| format!("test_{}.js", i)) @@ -1503,11 +1525,12 @@ mod tests { "test_5".to_string() => build_mock_test_runtime("test_5.js", 34.0), }, }; - let gen_resmoke_service = build_mocked_service(test_list, task_history.clone(), n_suites); + let gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); let params = ResmokeGenParams { task_name: "my_task".to_string(), require_multiversion_generate_tasks: false, + num_tasks, ..Default::default() }; @@ -1517,12 +1540,12 @@ mod tests { .unwrap(); assert_eq!(suite.display_name(), "my_task".to_string()); - assert_eq!(suite.sub_tasks().len(), n_suites); + assert_eq!(suite.sub_tasks().len(), num_tasks); } #[tokio::test] async fn test_generate_resmoke_tasks_multiversion_success() { - let n_suites = 3; + let num_tasks = 3; let test_list = vec![ "test_0.js".to_string(), "test_1.js".to_string(), @@ -1533,7 +1556,7 @@ mod tests { task_name: "my_task".to_string(), test_map: hashmap! {}, }; - let gen_resmoke_service = build_mocked_service(test_list, task_history.clone(), n_suites); + let gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); let generate_tasks = vec![ MultiversionGenerateTaskConfig { @@ -1549,6 +1572,7 @@ mod tests { task_name: "my_task".to_string(), multiversion_generate_tasks: Some(generate_tasks.clone()), require_multiversion_generate_tasks: true, + num_tasks, ..Default::default() }; @@ -1558,13 +1582,13 @@ mod tests { .unwrap(); assert_eq!(suite.display_name(), "my_task".to_string()); - assert_eq!(suite.sub_tasks().len(), n_suites * generate_tasks.len()); + assert_eq!(suite.sub_tasks().len(), num_tasks * generate_tasks.len()); } #[tokio::test] #[should_panic] async fn test_generate_resmoke_tasks_multiversion_fail() { - let n_suites = 3; + let num_tasks = 3; let test_list = vec![ "test_0.js".to_string(), "test_1.js".to_string(), @@ -1575,12 +1599,13 @@ mod tests { task_name: "my_task".to_string(), test_map: hashmap! {}, }; - let gen_resmoke_service = build_mocked_service(test_list, task_history.clone(), n_suites); + let gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); let params = ResmokeGenParams { task_name: "my_task".to_string(), multiversion_generate_tasks: None, require_multiversion_generate_tasks: true, + num_tasks, ..Default::default() }; diff --git a/tests/data/evergreen.yml b/tests/data/evergreen.yml index 188d996..d5995a7 100644 --- a/tests/data/evergreen.yml +++ b/tests/data/evergreen.yml @@ -4211,6 +4211,7 @@ tasks: use_large_distro: "true" resmoke_jobs_max: 1 resmoke_repeat_suites: 5 + num_tasks: 10 - <<: *task_template name: aggregation_sharded_collections_passthrough @@ -4242,6 +4243,8 @@ tasks: tags: ["auth"] commands: - func: "generate resmoke tasks" + vars: + num_tasks: 2 - name: burn_in_tags_gen tags: [] diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 31ab731..a14b38a 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -31,7 +31,7 @@ fn test_end2end_execution() { assert!(tmp_dir_path.exists()); let files = std::fs::read_dir(tmp_dir_path).unwrap(); - assert_eq!(686, files.into_iter().collect::>().len()); + assert_eq!(688, files.into_iter().collect::>().len()); } #[test] From 20244d0209c6be31aabbd0a41e7dc6fe561f2a3c Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Fri, 18 Apr 2025 14:45:02 +0000 Subject: [PATCH 2/3] fix comment --- src/task_types/resmoke_tasks.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/task_types/resmoke_tasks.rs b/src/task_types/resmoke_tasks.rs index f771ed5..6711c06 100644 --- a/src/task_types/resmoke_tasks.rs +++ b/src/task_types/resmoke_tasks.rs @@ -77,7 +77,7 @@ pub struct ResmokeGenParams { pub platform: Option, /// Name of variant specific suffix to add to tasks pub gen_task_suffix: Option, - /// Number of sub-tasks fuzzer should generate. + /// Number of sub-tasks to generate. pub num_tasks: usize, } @@ -1365,7 +1365,7 @@ mod tests { let sub_suites = gen_resmoke_service .split_task_fallback(¶ms, None, None) - .unwrap(); + .unwrap();n assert_eq!(sub_suites.len(), 0); } From 54648d2e8b0afc1b5c0bc4eec817d13a10cac90d Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Fri, 18 Apr 2025 14:50:18 +0000 Subject: [PATCH 3/3] remove typo --- src/task_types/resmoke_tasks.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/task_types/resmoke_tasks.rs b/src/task_types/resmoke_tasks.rs index 6711c06..c93a477 100644 --- a/src/task_types/resmoke_tasks.rs +++ b/src/task_types/resmoke_tasks.rs @@ -1365,7 +1365,7 @@ mod tests { let sub_suites = gen_resmoke_service .split_task_fallback(¶ms, None, None) - .unwrap();n + .unwrap(); assert_eq!(sub_suites.len(), 0); }