From 21bca3c1bc46c98f5fcda17ae6b34a1008081e11 Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Wed, 23 Apr 2025 14:40:47 +0000 Subject: [PATCH 1/6] Split large tasks on required variants based on total test runtime --- src/evergreen/evg_config.rs | 2 +- src/lib.rs | 25 ++- src/main.rs | 30 ++- src/services/config_extraction.rs | 8 +- src/task_types/burn_in_tests.rs | 2 +- src/task_types/resmoke_tasks.rs | 356 ++++++++++++++++++++++-------- 6 files changed, 318 insertions(+), 105 deletions(-) diff --git a/src/evergreen/evg_config.rs b/src/evergreen/evg_config.rs index 5519d8a..6f68daa 100644 --- a/src/evergreen/evg_config.rs +++ b/src/evergreen/evg_config.rs @@ -4,7 +4,7 @@ use std::{collections::HashMap, path::Path, process::Command}; use shrub_rs::models::{project::EvgProject, task::EvgTask, variant::BuildVariant}; -const REQUIRED_PREFIX: &str = "!"; +use crate::REQUIRED_PREFIX; pub trait EvgConfigService: Sync + Send { /// Get a map of build variant names to build variant definitions. diff --git a/src/lib.rs b/src/lib.rs index d72c65e..73ca4e7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -58,7 +58,7 @@ mod utils; const BURN_IN_TESTS_PREFIX: &str = "burn_in_tests"; const BURN_IN_TASKS_PREFIX: &str = "burn_in_tasks"; const BURN_IN_BV_SUFFIX: &str = "generated-by-burn-in-tags"; -const DEFAULT_SUB_TASKS_PER_TASK: usize = 5; +const REQUIRED_PREFIX: &str = "!"; type GenTaskCollection = HashMap>; @@ -145,6 +145,24 @@ pub struct ExecutionConfiguration<'a> { pub burn_in_tests_command: &'a str, /// S3 endpoint to get test stats from. pub s3_test_stats_endpoint: &'a str, + pub subtask_limits: SubtaskLimits, +} + +#[derive(Debug, Clone)] +pub struct SubtaskLimits { + // Ideal total test runtime (in seconds) for individual subtasks on required + // variants, used to determine the number of subtasks for tasks on required variants. + pub test_runtime_per_required_subtask: f64, + + // Threshold of total test runtime (in seconds) for a required task to be considered + // large enough to warrant splitting into more that the default number of tasks. + pub large_required_task_runtime_threshold: f64, + + // Default number of subtasks that should be generated for tasks + pub default_subtasks_per_task: usize, + + // Maximum number of subtasks that can be generated for tasks + pub max_subtasks_per_task: usize, } /// Collection of services needed to execution. @@ -215,6 +233,7 @@ impl Dependencies { multiversion_service, fs_service, gen_resmoke_config, + execution_config.subtask_limits, )); let gen_task_service = Arc::new(GenerateTasksServiceImpl::new( evg_config_service, @@ -589,7 +608,7 @@ impl GenerateTasksService for GenerateTasksServiceImpl { )?; Some( self.gen_resmoke_service - .generate_resmoke_task(¶ms, &build_variant.name) + .generate_resmoke_task(¶ms, build_variant) .await?, ) }; @@ -1021,7 +1040,7 @@ mod tests { async fn generate_resmoke_task( &self, _params: &ResmokeGenParams, - _build_variant: &str, + _build_variant: &BuildVariant, ) -> Result> { todo!() } diff --git a/src/main.rs b/src/main.rs index e7cb10e..782465a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,7 +7,7 @@ use std::{ use anyhow::Result; use clap::Parser; use mongo_task_generator::{ - generate_configuration, Dependencies, ExecutionConfiguration, ProjectInfo, + generate_configuration, Dependencies, ExecutionConfiguration, ProjectInfo, SubtaskLimits, }; use serde::Deserialize; use tracing::{error, event, Level}; @@ -19,6 +19,10 @@ const DEFAULT_RESMOKE_COMMAND: &str = "python buildscripts/resmoke.py"; const DEFAULT_BURN_IN_TESTS_COMMAND: &str = "python buildscripts/burn_in_tests.py run"; const DEFAULT_TARGET_DIRECTORY: &str = "generated_resmoke_config"; const DEFAULT_S3_TEST_STATS_ENDPOINT: &str = "https://mongo-test-stats.s3.amazonaws.com"; +const DEFAULT_MAX_SUBTASKS_PER_TASK: &str = "10"; +const DEFAULT_DEFAULT_SUBTASKS_PER_TASKS: &str = "5"; +const DEFAULT_TEST_RUNTIME_PER_REQUIRED_SUBTASK: &str = "3600"; +const DEFAULT_LARGE_REQUIRED_TASK_RUNTIME_THRESHOLD: &str = "7200"; /// Expansions from evergreen to determine settings for how task should be generated. #[derive(Debug, Deserialize)] @@ -129,6 +133,24 @@ struct Args { /// S3 endpoint to get test stats from. #[clap(long, default_value = DEFAULT_S3_TEST_STATS_ENDPOINT)] s3_test_stats_endpoint: String, + + // Ideal total test runtime (in seconds) for individual subtasks on required + // variants, used to determine the number of subtasks for tasks on required variants. + #[clap(long, default_value = DEFAULT_TEST_RUNTIME_PER_REQUIRED_SUBTASK)] + test_runtime_per_required_subtask: f64, + + // Threshold of total test runtime (in seconds) for a required task to be considered + // large enough to warrant splitting into more that the default number of tasks. + #[clap(long, default_value = DEFAULT_LARGE_REQUIRED_TASK_RUNTIME_THRESHOLD)] + large_required_task_runtime_threshold: f64, + + // Default number of subtasks that should be generated for tasks + #[clap(long, default_value = DEFAULT_DEFAULT_SUBTASKS_PER_TASKS)] + default_subtasks_per_task: usize, + + // Maximum number of subtasks that can be generated for tasks + #[clap(long, default_value = DEFAULT_MAX_SUBTASKS_PER_TASK)] + max_subtasks_per_task: usize, } /// Configure logging for the command execution. @@ -165,6 +187,12 @@ async fn main() { include_fully_disabled_feature_tests: args.include_fully_disabled_feature_tests, burn_in_tests_command: &args.burn_in_tests_command, s3_test_stats_endpoint: &args.s3_test_stats_endpoint, + subtask_limits: SubtaskLimits { + test_runtime_per_required_subtask: args.test_runtime_per_required_subtask, + max_subtasks_per_task: args.max_subtasks_per_task, + default_subtasks_per_task: args.default_subtasks_per_task, + large_required_task_runtime_threshold: args.large_required_task_runtime_threshold, + }, }; let deps = Dependencies::new(execution_config).unwrap(); diff --git a/src/services/config_extraction.rs b/src/services/config_extraction.rs index c0a9cc3..93b670f 100644 --- a/src/services/config_extraction.rs +++ b/src/services/config_extraction.rs @@ -18,7 +18,6 @@ use crate::{ multiversion::MultiversionService, resmoke_tasks::ResmokeGenParams, }, utils::task_name::remove_gen_suffix, - DEFAULT_SUB_TASKS_PER_TASK, }; /// Interface for performing extractions of evergreen project configuration. @@ -249,13 +248,10 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl { .evg_config_utils .lookup_build_variant_expansion(UNIQUE_GEN_SUFFIX_EXPANSION, variant); } - let num_tasks = match self + let num_tasks = self .evg_config_utils .get_gen_task_var(task_def, "num_tasks") - { - Some(str) => str.parse().unwrap(), - _ => DEFAULT_SUB_TASKS_PER_TASK, - }; + .map(|str| str.parse().unwrap()); Ok(ResmokeGenParams { task_name, diff --git a/src/task_types/burn_in_tests.rs b/src/task_types/burn_in_tests.rs index 8c28098..2e54bf2 100644 --- a/src/task_types/burn_in_tests.rs +++ b/src/task_types/burn_in_tests.rs @@ -630,7 +630,7 @@ mod tests { async fn generate_resmoke_task( &self, _params: &ResmokeGenParams, - _build_variant: &str, + _build_variant: &BuildVariant, ) -> Result> { todo!() } diff --git a/src/task_types/resmoke_tasks.rs b/src/task_types/resmoke_tasks.rs index c93a477..acffdb4 100644 --- a/src/task_types/resmoke_tasks.rs +++ b/src/task_types/resmoke_tasks.rs @@ -14,6 +14,7 @@ use shrub_rs::models::{ commands::{fn_call, fn_call_with_params, EvgCommand}, params::ParamValue, task::{EvgTask, TaskDependency}, + variant::BuildVariant, }; use tokio::sync::Mutex; use tracing::{event, warn, Level}; @@ -33,7 +34,7 @@ use crate::{ }, resmoke::resmoke_proxy::TestDiscovery, utils::{fs_service::FsService, task_name::name_generated_task}, - DEFAULT_SUB_TASKS_PER_TASK, + SubtaskLimits, REQUIRED_PREFIX, }; use super::{ @@ -43,7 +44,7 @@ use super::{ }; /// Parameters describing how a specific resmoke suite should be generated. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct ResmokeGenParams { /// Name of task being generated. pub task_name: String, @@ -77,32 +78,8 @@ pub struct ResmokeGenParams { pub platform: Option, /// Name of variant specific suffix to add to tasks pub gen_task_suffix: Option, - /// Number of sub-tasks to generate. - pub num_tasks: usize, -} - -impl Default for ResmokeGenParams { - fn default() -> ResmokeGenParams { - ResmokeGenParams { - num_tasks: DEFAULT_SUB_TASKS_PER_TASK, - task_name: Default::default(), - multiversion_generate_tasks: Default::default(), - suite_name: Default::default(), - use_large_distro: Default::default(), - use_xlarge_distro: Default::default(), - require_multiversion_setup: Default::default(), - require_multiversion_generate_tasks: Default::default(), - repeat_suites: Default::default(), - resmoke_args: Default::default(), - resmoke_jobs_max: Default::default(), - config_location: Default::default(), - dependencies: Default::default(), - is_enterprise: Default::default(), - pass_through_vars: Default::default(), - platform: Default::default(), - gen_task_suffix: Default::default(), - } - } + /// Number of sub-tasks requested in the task's Evergreen YAML definition. + pub num_tasks: Option, } impl ResmokeGenParams { @@ -294,7 +271,7 @@ pub trait GenResmokeTaskService: Sync + Send { async fn generate_resmoke_task( &self, params: &ResmokeGenParams, - build_variant: &str, + build_variant: &BuildVariant, ) -> Result>; /// Build a shrub task to execute a split sub-task. @@ -366,6 +343,8 @@ pub struct GenResmokeTaskServiceImpl { /// Configuration for generating resmoke tasks. config: GenResmokeConfig, + + subtask_limits: SubtaskLimits, } impl GenResmokeTaskServiceImpl { @@ -388,6 +367,7 @@ impl GenResmokeTaskServiceImpl { multiversion_service: Arc, fs_service: Arc, config: GenResmokeConfig, + subtask_limits: SubtaskLimits, ) -> Self { Self { task_history_service, @@ -396,6 +376,7 @@ impl GenResmokeTaskServiceImpl { multiversion_service, fs_service, config, + subtask_limits, } } } @@ -409,6 +390,7 @@ impl GenResmokeTaskServiceImpl { /// * `task_stats` - Statistics on the historic runtimes of tests in the task. /// * `multiversion_name` - Name of task if performing multiversion generation. /// * `multiversion_tags` - Tag to include when performing multiversion generation. + /// * `build_variant` - Build variant to base generation off of. /// /// # Returns /// @@ -419,16 +401,43 @@ impl GenResmokeTaskServiceImpl { task_stats: &TaskRuntimeHistory, multiversion_name: Option<&str>, multiversion_tags: Option, + build_variant: &BuildVariant, ) -> Result> { let origin_suite = multiversion_name.unwrap_or(¶ms.suite_name); let test_list = self.get_test_list(params, multiversion_name)?; let total_runtime = task_stats .test_map .iter() + .filter(|(_, history)| test_list.contains(&history.test_name)) .fold(0.0, |init, (_, item)| init + item.average_runtime); - let max_tasks = min(params.num_tasks, test_list.len()); - let runtime_per_subtask = total_runtime / max_tasks as f64; + let ideal_num_tasks = match params.num_tasks { + Some(t) => t, + None if build_variant + .display_name + .as_ref() + .unwrap() + .starts_with(REQUIRED_PREFIX) + && total_runtime > self.subtask_limits.large_required_task_runtime_threshold => + { + self.subtask_limits.default_subtasks_per_task + + ((total_runtime - self.subtask_limits.large_required_task_runtime_threshold) + / self.subtask_limits.test_runtime_per_required_subtask) + as usize + } + None => self.subtask_limits.default_subtasks_per_task, + }; + + let num_tasks = *[ + ideal_num_tasks, + test_list.len(), + self.subtask_limits.max_subtasks_per_task, + ] + .iter() + .min() + .unwrap(); + + let runtime_per_subtask = total_runtime / num_tasks as f64; event!( Level::INFO, "Splitting task: {}, runtime: {}, tests: {}", @@ -438,8 +447,8 @@ impl GenResmokeTaskServiceImpl { ); let sorted_test_list = sort_tests_by_runtime(test_list, task_stats); - let mut running_tests = vec![vec![]; max_tasks]; - let mut running_runtimes = vec![0.0; max_tasks]; + let mut running_tests = vec![vec![]; num_tasks]; + let mut running_runtimes = vec![0.0; num_tasks]; let mut left_tests = vec![]; for test in sorted_test_list { @@ -455,7 +464,7 @@ impl GenResmokeTaskServiceImpl { let min_idx = get_min_index(&running_runtimes); for (i, test) in left_tests.iter().enumerate() { - running_tests[(min_idx + i) % max_tasks].push(test.clone()); + running_tests[(min_idx + i) % num_tasks].push(test.clone()); } let mut sub_suites = vec![]; @@ -536,7 +545,12 @@ impl GenResmokeTaskServiceImpl { return Ok(sub_suites); } - let n = min(test_list.len(), params.num_tasks); + let requested_num_tasks = match params.num_tasks { + Some(tasks) => tasks, + None => self.subtask_limits.default_subtasks_per_task, + }; + + let n = min(test_list.len(), requested_num_tasks); let len = test_list.len(); let (quo, rem) = (len / n, len % n); let split = (quo + 1) * rem; @@ -571,7 +585,7 @@ impl GenResmokeTaskServiceImpl { async fn create_multiversion_tasks( &self, params: &ResmokeGenParams, - build_variant: &str, + build_variant: &BuildVariant, ) -> Result> { let mut mv_sub_suites = vec![]; for multiversion_task in params.multiversion_generate_tasks.as_ref().unwrap() { @@ -594,7 +608,7 @@ impl GenResmokeTaskServiceImpl { /// # Arguments /// /// * `params` - Parameters for how tasks should be generated. - /// * `build_variant` - Name of build variant to base generation off. + /// * `build_variant` - Build variant to base generation off of. /// * `multiversion_name` - Name of task if performing multiversion generation. /// * `multiversion_tags` - Tag to include when performing multiversion generation. /// @@ -604,7 +618,7 @@ impl GenResmokeTaskServiceImpl { async fn create_tasks( &self, params: &ResmokeGenParams, - build_variant: &str, + build_variant: &BuildVariant, multiversion_name: Option<&str>, multiversion_tags: Option, ) -> Result> { @@ -613,7 +627,7 @@ impl GenResmokeTaskServiceImpl { } else { let task_history = self .task_history_service - .get_task_history(¶ms.task_name, build_variant) + .get_task_history(¶ms.task_name, &build_variant.name) .await; match task_history { @@ -622,10 +636,11 @@ impl GenResmokeTaskServiceImpl { &task_history, multiversion_name, multiversion_tags.clone(), + build_variant, )?, Err(err) => { warn!( - build_variant = build_variant, + build_variant = build_variant.name, task_name = params.task_name.as_str(), error = err.to_string().as_str(), "Could not get task history from S3", @@ -716,7 +731,7 @@ impl GenResmokeTaskService for GenResmokeTaskServiceImpl { async fn generate_resmoke_task( &self, params: &ResmokeGenParams, - build_variant: &str, + build_variant: &BuildVariant, ) -> Result> { let sub_suites = if params.require_multiversion_generate_tasks { self.create_multiversion_tasks(params, build_variant) @@ -1143,6 +1158,12 @@ mod tests { Arc::new(multiversion_service), Arc::new(fs_service), config, + SubtaskLimits { + test_runtime_per_required_subtask: 3600.0, + large_required_task_runtime_threshold: 7200.0, + default_subtasks_per_task: 5, + max_subtasks_per_task: 10, + }, ) } @@ -1153,7 +1174,6 @@ mod tests { hooks: vec![], } } - #[test] fn test_split_task_should_split_tasks_by_runtime() { // In this test we will create 3 subtasks with 6 tests. The first sub task should contain @@ -1178,12 +1198,21 @@ mod tests { let gen_resmoke_service = build_mocked_service(test_list.clone(), task_history.clone()); let params = ResmokeGenParams { - num_tasks, + num_tasks: Some(num_tasks), ..Default::default() }; let sub_suites = gen_resmoke_service - .split_task(¶ms, &task_history, None, None) + .split_task( + ¶ms, + &task_history, + None, + None, + &BuildVariant { + display_name: Some("build-variant".to_string()), + ..Default::default() + }, + ) .unwrap(); assert_eq!(sub_suites.len(), num_tasks); @@ -1197,7 +1226,6 @@ mod tests { assert!(suite_2.test_list.contains(&"test_3.js".to_string())); assert!(suite_2.test_list.contains(&"test_5.js".to_string())); } - #[test] fn test_split_task_with_missing_history_should_split_tasks_equally() { let num_tasks = 3; @@ -1216,12 +1244,21 @@ mod tests { let gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); let params = ResmokeGenParams { - num_tasks, + num_tasks: Some(num_tasks), ..Default::default() }; let sub_suites = gen_resmoke_service - .split_task(¶ms, &task_history, None, None) + .split_task( + ¶ms, + &task_history, + None, + None, + &BuildVariant { + display_name: Some("build-variant".to_string()), + ..Default::default() + }, + ) .unwrap(); assert_eq!(sub_suites.len(), num_tasks); @@ -1232,7 +1269,6 @@ mod tests { let suite_2 = &sub_suites[2]; assert_eq!(suite_2.test_list.len(), 4); } - #[test] fn test_split_tasks_should_include_multiversion_information() { let num_tasks = 3; @@ -1251,7 +1287,7 @@ mod tests { let gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); let params = ResmokeGenParams { - num_tasks, + num_tasks: Some(num_tasks), ..Default::default() }; @@ -1261,6 +1297,10 @@ mod tests { &task_history, Some("multiversion_test"), Some("multiversion_tag".to_string()), + &BuildVariant { + display_name: Some("build-variant".to_string()), + ..Default::default() + }, ) .unwrap(); @@ -1273,9 +1313,7 @@ mod tests { ); } } - // split_task_fallback tests - #[test] fn test_split_task_fallback_should_split_tasks_count() { let num_tasks = 3; @@ -1291,19 +1329,17 @@ mod tests { let gen_resmoke_service = build_mocked_service(test_list.clone(), task_history.clone()); let params = ResmokeGenParams { - num_tasks, + num_tasks: Some(num_tasks), ..Default::default() }; let sub_suites = gen_resmoke_service .split_task_fallback(¶ms, None, None) .unwrap(); - assert_eq!(sub_suites.len(), num_tasks); for sub_suite in &sub_suites { assert_eq!(sub_suite.test_list.len(), n_tests / num_tasks); } - let all_tests: Vec = sub_suites .iter() .flat_map(|s| s.test_list.clone()) @@ -1313,7 +1349,6 @@ mod tests { assert!(all_tests.contains(&test_name.to_string())); } } - #[test] fn test_split_task_fallback_has_remainder() { let num_tasks = 3; @@ -1329,14 +1364,13 @@ mod tests { let gen_resmoke_service = build_mocked_service(test_list.clone(), task_history.clone()); let params = ResmokeGenParams { - num_tasks, + num_tasks: Some(num_tasks), ..Default::default() }; let sub_suites = gen_resmoke_service .split_task_fallback(¶ms, None, None) .unwrap(); - assert_eq!(sub_suites.len(), num_tasks); let all_tests: Vec = sub_suites .iter() @@ -1350,26 +1384,22 @@ mod tests { #[test] fn test_split_task_fallback_empty_suite() { - let num_tasks = 1; + let num_tasks = Some(1); let test_list = vec![]; let task_history = TaskRuntimeHistory { task_name: "my task".to_string(), test_map: hashmap! {}, }; let gen_resmoke_service = build_mocked_service(test_list.clone(), task_history.clone()); - let params = ResmokeGenParams { num_tasks, ..Default::default() }; - let sub_suites = gen_resmoke_service .split_task_fallback(¶ms, None, None) .unwrap(); - assert_eq!(sub_suites.len(), 0); } - // tests for get_test_list. #[rstest] #[case(true, 12)] @@ -1378,7 +1408,7 @@ mod tests { #[case] is_enterprise: bool, #[case] expected_tests: usize, ) { - let num_tasks = 3; + let num_tasks = Some(3); let mut test_list: Vec = (0..6) .into_iter() .map(|i| format!("test_{}.js", i)) @@ -1394,13 +1424,11 @@ mod tests { test_map: hashmap! {}, }; let gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); - let params = ResmokeGenParams { is_enterprise, num_tasks, ..Default::default() }; - let sub_suites = gen_resmoke_service .split_task_fallback(¶ms, None, None) .unwrap(); @@ -1410,7 +1438,6 @@ mod tests { .collect(); assert_eq!(expected_tests, all_tests.len()); } - #[rstest] #[case(true, 12)] #[case(false, 12)] @@ -1418,7 +1445,7 @@ mod tests { #[case] is_enterprise: bool, #[case] expected_tests: usize, ) { - let num_tasks = 3; + let num_tasks = Some(3); let mut test_list: Vec = (0..6) .into_iter() .map(|i| format!("test_{}.js", i)) @@ -1435,13 +1462,11 @@ mod tests { }; let mut gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); gen_resmoke_service.config.enterprise_dir = None; - let params = ResmokeGenParams { is_enterprise, num_tasks, ..Default::default() }; - let sub_suites = gen_resmoke_service .split_task_fallback(¶ms, None, None) .unwrap(); @@ -1451,7 +1476,6 @@ mod tests { .collect(); assert_eq!(expected_tests, all_tests.len()); } - // create_multiversion_combinations tests. #[tokio::test] async fn test_create_multiversion_tasks() { @@ -1466,7 +1490,7 @@ mod tests { old_version: "last-continuous".to_string(), }, ]), - num_tasks: 1, + num_tasks: Some(1), ..Default::default() }; let task_history = TaskRuntimeHistory { @@ -1482,7 +1506,13 @@ mod tests { let gen_resmoke_service = build_mocked_service(test_list.clone(), task_history); let suite_list = gen_resmoke_service - .create_multiversion_tasks(¶ms, "build_variant") + .create_multiversion_tasks( + ¶ms, + &BuildVariant { + display_name: Some("build-variant".to_string()), + ..Default::default() + }, + ) .await .unwrap(); @@ -1502,7 +1532,6 @@ mod tests { .iter() .all(|test| test_list.contains(test))); } - // generate_resmoke_task tests. #[tokio::test] async fn test_generate_resmoke_tasks_standard() { @@ -1526,16 +1555,21 @@ mod tests { }, }; let gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); - let params = ResmokeGenParams { task_name: "my_task".to_string(), require_multiversion_generate_tasks: false, - num_tasks, + num_tasks: Some(num_tasks), ..Default::default() }; let suite = gen_resmoke_service - .generate_resmoke_task(¶ms, "build-variant") + .generate_resmoke_task( + ¶ms, + &BuildVariant { + display_name: Some("build-variant".to_string()), + ..Default::default() + }, + ) .await .unwrap(); @@ -1543,6 +1577,143 @@ mod tests { assert_eq!(suite.sub_tasks().len(), num_tasks); } + #[tokio::test] + async fn test_generate_resmoke_tasks_required_variant_large() { + // Creating tasks based off of a required build variant with large total test runtime, + // more than the default number of tasks should be used. + + let test_list: Vec = (0..8) + .into_iter() + .map(|i| format!("test_{}.js", i)) + .collect(); + let task_history = TaskRuntimeHistory { + task_name: "my_task".to_string(), + test_map: hashmap! { + "test_0".to_string() => build_mock_test_runtime("test_0.js", 1800.0), + "test_1".to_string() => build_mock_test_runtime("test_1.js", 1800.0), + "test_2".to_string() => build_mock_test_runtime("test_2.js", 1800.0), + "test_4".to_string() => build_mock_test_runtime("test_4.js", 1800.0), + "test_5".to_string() => build_mock_test_runtime("test_5.js", 1800.0), + "test_6".to_string() => build_mock_test_runtime("test_6.js", 1800.0), + "test_7".to_string() => build_mock_test_runtime("test_7.js", 1800.0), + "test_8".to_string() => build_mock_test_runtime("test_8.js", 1800.0), + }, + }; + let gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); + + let params = ResmokeGenParams { + task_name: "my_task".to_string(), + require_multiversion_generate_tasks: false, + ..Default::default() + }; + + let suite = gen_resmoke_service + .generate_resmoke_task( + ¶ms, + &BuildVariant { + display_name: Some("! required build variant".to_string()), + ..Default::default() + }, + ) + .await + .unwrap(); + + assert_eq!(suite.display_name(), "my_task".to_string()); + assert_eq!(suite.sub_tasks().len(), 6); + } + + #[tokio::test] + async fn test_generate_resmoke_tasks_required_variant_medium() { + // Creating tasks based off of a required build variant with large total test runtime, + // more than the default number of tasks should be used. + + let test_list: Vec = (0..8) + .into_iter() + .map(|i| format!("test_{}.js", i)) + .collect(); + let task_history = TaskRuntimeHistory { + task_name: "my_task".to_string(), + test_map: hashmap! { + "test_0".to_string() => build_mock_test_runtime("test_0.js", 900.0), + "test_1".to_string() => build_mock_test_runtime("test_1.js", 900.0), + "test_2".to_string() => build_mock_test_runtime("test_2.js", 900.0), + "test_3".to_string() => build_mock_test_runtime("test_3.js", 900.0), + "test_4".to_string() => build_mock_test_runtime("test_4.js", 900.0), + "test_5".to_string() => build_mock_test_runtime("test_5.js", 900.0), + "test_6".to_string() => build_mock_test_runtime("test_6.js", 900.0), + "test_7".to_string() => build_mock_test_runtime("test_7.js", 900.0), + "test_8".to_string() => build_mock_test_runtime("test_8.js", 900.0), + }, + }; + let gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); + + let params = ResmokeGenParams { + task_name: "my_task".to_string(), + require_multiversion_generate_tasks: false, + ..Default::default() + }; + + let suite = gen_resmoke_service + .generate_resmoke_task( + ¶ms, + &BuildVariant { + display_name: Some("! required build variant".to_string()), + ..Default::default() + }, + ) + .await + .unwrap(); + + assert_eq!(suite.display_name(), "my_task".to_string()); + assert_eq!(suite.sub_tasks().len(), 5); + } + + #[tokio::test] + async fn test_generate_resmoke_tasks_required_variant_small() { + // Creating tasks based off of a required build variant with small test runtime, + // the default number of subtasks should be used. + + let test_list: Vec = (0..9) + .into_iter() + .map(|i| format!("test_{}.js", i)) + .collect(); + let task_history = TaskRuntimeHistory { + task_name: "my_task".to_string(), + test_map: hashmap! { + "test_0".to_string() => build_mock_test_runtime("test_0.js", 1.0), + "test_1".to_string() => build_mock_test_runtime("test_1.js", 1.0), + "test_2".to_string() => build_mock_test_runtime("test_2.js", 1.0), + "test_3".to_string() => build_mock_test_runtime("test_3.js", 1.0), + "test_4".to_string() => build_mock_test_runtime("test_4.js", 1.0), + "test_5".to_string() => build_mock_test_runtime("test_5.js", 1.0), + "test_6".to_string() => build_mock_test_runtime("test_6.js", 1.0), + "test_7".to_string() => build_mock_test_runtime("test_7.js", 1.0), + "test_8".to_string() => build_mock_test_runtime("test_8.js", 1.0), + }, + }; + let gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); + + let params = ResmokeGenParams { + task_name: "my_task".to_string(), + require_multiversion_generate_tasks: false, + ..Default::default() + }; + + let suite = gen_resmoke_service + .generate_resmoke_task( + ¶ms, + &BuildVariant { + display_name: Some("! required build variant".to_string()), + ..Default::default() + }, + ) + .await + .unwrap(); + + assert_eq!(suite.display_name(), "my_task".to_string()); + assert_eq!(suite.sub_tasks().len(), 5); + } + #[tokio::test] async fn test_generate_resmoke_tasks_multiversion_success() { let num_tasks = 3; @@ -1557,7 +1728,6 @@ mod tests { test_map: hashmap! {}, }; let gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); - let generate_tasks = vec![ MultiversionGenerateTaskConfig { suite_name: "suite1_last_lts".to_string(), @@ -1572,23 +1742,28 @@ mod tests { task_name: "my_task".to_string(), multiversion_generate_tasks: Some(generate_tasks.clone()), require_multiversion_generate_tasks: true, - num_tasks, + num_tasks: Some(num_tasks), ..Default::default() }; let suite = gen_resmoke_service - .generate_resmoke_task(¶ms, "build-variant") + .generate_resmoke_task( + ¶ms, + &BuildVariant { + display_name: Some("build-variant".to_string()), + ..Default::default() + }, + ) .await .unwrap(); assert_eq!(suite.display_name(), "my_task".to_string()); assert_eq!(suite.sub_tasks().len(), num_tasks * generate_tasks.len()); } - #[tokio::test] #[should_panic] async fn test_generate_resmoke_tasks_multiversion_fail() { - let num_tasks = 3; + let num_tasks = Some(3); let test_list = vec![ "test_0.js".to_string(), "test_1.js".to_string(), @@ -1600,7 +1775,6 @@ mod tests { test_map: hashmap! {}, }; let gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); - let params = ResmokeGenParams { task_name: "my_task".to_string(), multiversion_generate_tasks: None, @@ -1610,11 +1784,16 @@ mod tests { }; gen_resmoke_service - .generate_resmoke_task(¶ms, "build-variant") + .generate_resmoke_task( + ¶ms, + &BuildVariant { + display_name: Some("build-variant".to_string()), + ..Default::default() + }, + ) .await .unwrap(); } - // resmoke_commands tests. fn get_evg_fn_name(evg_command: &EvgCommand) -> Option<&str> { if let EvgCommand::Function(func) = evg_command { @@ -1623,26 +1802,21 @@ mod tests { None } } - #[test] fn test_resmoke_commands() { let commands = resmoke_commands("run test", hashmap! {}, false); - assert_eq!(commands.len(), 3); assert_eq!(get_evg_fn_name(&commands[0]), Some("do setup")); assert_eq!(get_evg_fn_name(&commands[2]), Some("run test")); } - #[test] fn test_resmoke_commands_should_include_multiversion() { let commands = resmoke_commands("run test", hashmap! {}, true); - assert_eq!(commands.len(), 6); assert_eq!(get_evg_fn_name(&commands[2]), Some("do setup")); assert_eq!(get_evg_fn_name(&commands[4]), Some("do multiversion setup")); assert_eq!(get_evg_fn_name(&commands[5]), Some("run test")); } - // sort_tests_by_runtime tests. #[rstest] #[case(vec![100.0, 50.0, 30.0, 25.0, 20.0, 15.0], vec![0, 1, 2, 3, 4, 5])] @@ -1679,12 +1853,9 @@ mod tests { .into_iter() .map(|i| format!("test_{}.js", sorted_indexes[i])) .collect(); - let result = sort_tests_by_runtime(test_list, &task_stats); - assert_eq!(result, expected_result); } - // get_min_index tests. #[rstest] #[case(vec![100.0, 50.0, 30.0, 25.0, 20.0, 15.0], 5)] @@ -1692,7 +1863,6 @@ mod tests { #[case(vec![25.0, 50.0, 15.0, 30.0, 100.0, 20.0], 2)] fn test_get_min_index(#[case] running_runtimes: Vec, #[case] expected_min_idx: usize) { let min_idx = get_min_index(&running_runtimes); - assert_eq!(min_idx, expected_min_idx); } } From 9994a781e96da44919dc1607589917d22bf1653a Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Wed, 23 Apr 2025 14:52:11 +0000 Subject: [PATCH 2/6] use aws-sdk to improve reliability of fetching test stats --- Cargo.lock | 1256 ++++++++++++++++++----------- Cargo.toml | 7 +- README.md | 4 +- docs/generating_tasks.md | 6 +- src/evergreen/evg_task_history.rs | 126 +-- src/lib.rs | 41 +- src/main.rs | 16 +- src/task_types/resmoke_tasks.rs | 4 - 8 files changed, 868 insertions(+), 592 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da602ec..7091eb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,6 +39,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -144,15 +150,9 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn", ] -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "atomicwrites" version = "0.4.3" @@ -170,6 +170,381 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "aws-config" +version = "1.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d1c2c88936a73c699225d0bc00684a534166b0cebc2659c3cdf08de8edc64c" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand 2.1.0", + "hex", + "http 0.2.12", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee7643696e7fdd74c10f9eb42848a87fe469d35eae9c3323f80aa98f350baac" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand 2.1.0", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.72.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c7ce6d85596c4bcb3aba8ad5bb134b08e204c8a475c9999c1af9290f80aa8ad" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand 2.1.0", + "hex", + "hmac", + "http 0.2.12", + "http-body 0.4.6", + "lru", + "once_cell", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c54bab121fe1881a74c338c5f723d1592bf3b53167f80268a1274f404e1acc38" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c8234fd024f7ac61c4e44ea008029bde934250f371efe7d4a39708397b1080c" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba60e1d519d6f23a9df712c04fdeadd7872ac911c84b2f62a8bda92e129b7962" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bfe75fad52793ce6dec0dc3d4b1f388f038b5eb866c8d4d7f3a8e21b5ea5051" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.1.0", + "once_cell", + "p256", + "percent-encoding", + "ring", + "sha2", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa59d1327d8b5053c54bf2eaae63bf629ba9e904434d0835a28ed3c0ed0a614e" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f45a1c384d7a393026bc5f5c177105aa9fa68e4749653b985707ac27d77295" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "crc64fast-nvme", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "461e5e02f9864cba17cff30f007c2e37ade94d01e87cdb5204e44a84e6d38c17" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "623a51127f24c30776c8b374295f2df78d92517386f77ba30773f15a30ce1422" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "865f7050bbc7107a6c98a397a9fcd9413690c27fa718446967cf03b2d3ac517e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand 2.1.0", + "h2", + "http 0.2.12", + "http-body 0.4.6", + "http-body 1.0.1", + "httparse", + "hyper", + "hyper-rustls", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.1.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7b8a53819e42f10d0821f56da995e1470b199686a1809168db6ca485665f042" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.1.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbd0a668309ec1f66c0f6bda4840dd6d4796ae26d699ebc266d7cc95c6d040f" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.71" @@ -194,6 +569,12 @@ dependencies = [ "backtrace", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.21.7" @@ -206,6 +587,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.6.0" @@ -265,6 +656,16 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "camino" version = "1.1.7" @@ -297,7 +698,7 @@ dependencies = [ "dialoguer", "duct", "enable-ansi-support", - "env_logger 0.11.5", + "env_logger", "guppy", "itertools 0.12.1", "log", @@ -419,7 +820,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.74", + "syn", ] [[package]] @@ -428,32 +829,6 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" -[[package]] -name = "cmd_lib" -version = "1.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718f77610af91e4d648fe9da0150ae58698cbcfb93bf63e764907fbefd56ffe8" -dependencies = [ - "cmd_lib_macros", - "env_logger 0.10.2", - "faccess", - "lazy_static", - "log", - "os_pipe", -] - -[[package]] -name = "cmd_lib_macros" -version = "1.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a80fac05ed12fe97a70b5dfdd910c9b90b53f4de69002c3179e29ac2d066abc" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "color-eyre" version = "0.6.3" @@ -533,6 +908,30 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -542,6 +941,37 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4955638f00a809894c947f85a024020a20815b65a5eea633798ea7924edab2b3" +dependencies = [ + "crc", +] + +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -576,7 +1006,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn", ] [[package]] @@ -585,6 +1015,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffe7ed1d93f4553003e20b629abe9085e1e81b1429520f897f8f8860bc6dfc21" +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der" version = "0.7.9" @@ -595,6 +1035,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +dependencies = [ + "powerfmt", +] + [[package]] name = "dialoguer" version = "0.11.0" @@ -622,6 +1071,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -675,14 +1125,26 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve", + "rfc6979", + "signature 1.6.4", +] + [[package]] name = "ed25519" version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", - "signature", + "pkcs8 0.10.2", + "signature 2.2.0", ] [[package]] @@ -695,7 +1157,7 @@ dependencies = [ "ed25519", "serde", "sha2", - "signature", + "signature 2.2.0", "subtle", "zeroize", ] @@ -706,6 +1168,26 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der 0.6.1", + "digest", + "ff", + "generic-array", + "group", + "pkcs8 0.9.0", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "enable-ansi-support" version = "0.2.1" @@ -739,19 +1221,6 @@ dependencies = [ "log", ] -[[package]] -name = "env_logger" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" -dependencies = [ - "humantime", - "is-terminal", - "log", - "regex", - "termcolor", -] - [[package]] name = "env_logger" version = "0.11.5" @@ -788,17 +1257,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "faccess" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ae66425802d6a903e268ae1a08b8c38ba143520f227a205edf4e9c7e3e26d5" -dependencies = [ - "bitflags 1.3.2", - "libc", - "winapi", -] - [[package]] name = "fastrand" version = "1.9.0" @@ -814,6 +1272,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "fiat-crypto" version = "0.2.9" @@ -854,6 +1322,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -951,7 +1425,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn", ] [[package]] @@ -1007,10 +1481,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", - "js-sys", "libc", "wasi", - "wasm-bindgen", ] [[package]] @@ -1038,6 +1510,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "guppy" version = "0.17.7" @@ -1090,25 +1573,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "h2" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http 1.1.0", - "indexmap 2.4.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -1121,6 +1585,17 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "heck" version = "0.5.0" @@ -1133,6 +1608,21 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" version = "0.5.9" @@ -1236,7 +1726,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.3.26", + "h2", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -1250,26 +1740,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "h2 0.4.5", - "http 1.1.0", - "http-body 1.0.1", - "httparse", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - [[package]] name = "hyper-rustls" version = "0.24.2" @@ -1278,27 +1748,12 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.30", - "rustls 0.21.12", - "tokio", - "tokio-rustls 0.24.1", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" -dependencies = [ - "futures-util", - "http 1.1.0", - "hyper 1.4.1", - "hyper-util", - "rustls 0.23.12", - "rustls-pki-types", + "hyper", + "log", + "rustls", + "rustls-native-certs", "tokio", - "tokio-rustls 0.26.0", - "tower-service", + "tokio-rustls", ] [[package]] @@ -1308,48 +1763,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper 0.14.30", + "hyper", "native-tls", "tokio", "tokio-native-tls", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper 1.4.1", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - -[[package]] -name = "hyper-util" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "http 1.1.0", - "http-body 1.0.1", - "hyper 1.4.1", - "pin-project-lite", - "socket2", - "tokio", - "tower", - "tower-service", - "tracing", -] - [[package]] name = "iana-time-zone" version = "0.1.60" @@ -1430,9 +1849,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", ] [[package]] @@ -1526,7 +1942,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.6.0", "libc", - "redox_syscall 0.5.3", + "redox_syscall", ] [[package]] @@ -1557,12 +1973,31 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.2", +] + [[package]] name = "maplit" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.7.4" @@ -1597,7 +2032,7 @@ checksum = "dcf09caffaac8068c346b6df2a7fc27a177fd20b39421a39ce0a211bde679a6c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn", ] [[package]] @@ -1640,18 +2075,15 @@ dependencies = [ "anyhow", "assert_cmd", "async-trait", + "aws-config", + "aws-sdk-s3", "cargo-nextest", "chrono", "clap", - "cmd_lib", - "futures", "lazy_static", "maplit", "rand 0.8.5", "regex", - "reqwest 0.12.5", - "reqwest-middleware", - "reqwest-retry", "rstest", "serde", "serde_json", @@ -1851,6 +2283,21 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1904,7 +2351,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn", ] [[package]] @@ -1941,6 +2388,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "overload" version = "0.1.1" @@ -1963,38 +2416,24 @@ dependencies = [ ] [[package]] -name = "parking_lot" -version = "0.11.2" +name = "p256" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", + "ecdsa", + "elliptic-curve", + "sha2", ] [[package]] name = "parking_lot" version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" -dependencies = [ - "lock_api", - "parking_lot_core 0.9.10", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", ] [[package]] @@ -2005,7 +2444,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.3", + "redox_syscall", "smallvec", "windows-targets 0.52.6", ] @@ -2052,7 +2491,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn", ] [[package]] @@ -2067,14 +2506,24 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der 0.6.1", + "spki 0.6.0", +] + [[package]] name = "pkcs8" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.9", + "spki 0.7.3", ] [[package]] @@ -2089,6 +2538,12 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -2134,30 +2589,6 @@ dependencies = [ "toml_edit 0.21.1", ] -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - [[package]] name = "proc-macro2" version = "1.0.86" @@ -2282,15 +2713,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f705426858ccd7bbfe19798239d6b6bfd9bf96bde0624a84b92694046e98871" -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.5.3" @@ -2334,6 +2756,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.8.4" @@ -2366,12 +2794,12 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.3.26", + "h2", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.30", - "hyper-rustls 0.24.2", - "hyper-tls 0.5.0", + "hyper", + "hyper-rustls", + "hyper-tls", "ipnet", "js-sys", "log", @@ -2380,111 +2808,34 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.12", - "rustls-pemfile 1.0.4", + "rustls", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 0.1.2", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", - "tokio-rustls 0.24.1", + "tokio-rustls", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", "webpki-roots", - "winreg 0.50.0", -] - -[[package]] -name = "reqwest" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" -dependencies = [ - "base64 0.22.1", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2 0.4.5", - "http 1.1.0", - "http-body 1.0.1", - "http-body-util", - "hyper 1.4.1", - "hyper-rustls 0.27.2", - "hyper-tls 0.6.0", - "hyper-util", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls-pemfile 2.1.3", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper 1.0.1", - "system-configuration", - "tokio", - "tokio-native-tls", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg 0.52.0", -] - -[[package]] -name = "reqwest-middleware" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562ceb5a604d3f7c885a792d42c199fd8af239d0a51b2fa6a78aafa092452b04" -dependencies = [ - "anyhow", - "async-trait", - "http 1.1.0", - "reqwest 0.12.5", - "serde", - "thiserror", - "tower-service", -] - -[[package]] -name = "reqwest-retry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a83df1aaec00176d0fabb65dea13f832d2a446ca99107afc17c5d2d4981221d0" -dependencies = [ - "anyhow", - "async-trait", - "futures", - "getrandom", - "http 1.1.0", - "hyper 1.4.1", - "parking_lot 0.11.2", - "reqwest 0.12.5", - "reqwest-middleware", - "retry-policies", - "tokio", - "tracing", - "wasm-timer", + "winreg", ] [[package]] -name = "retry-policies" -version = "0.4.0" +name = "rfc6979" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" dependencies = [ - "rand 0.8.5", + "crypto-bigint 0.4.9", + "hmac", + "zeroize", ] [[package]] @@ -2541,7 +2892,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.74", + "syn", "unicode-ident", ] @@ -2581,21 +2932,20 @@ checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring", - "rustls-webpki 0.101.7", + "rustls-webpki", "sct", ] [[package]] -name = "rustls" -version = "0.23.12" +name = "rustls-native-certs" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ - "once_cell", - "rustls-pki-types", - "rustls-webpki 0.102.6", - "subtle", - "zeroize", + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", ] [[package]] @@ -2607,22 +2957,6 @@ dependencies = [ "base64 0.21.7", ] -[[package]] -name = "rustls-pemfile" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" -dependencies = [ - "base64 0.22.1", - "rustls-pki-types", -] - -[[package]] -name = "rustls-pki-types" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" - [[package]] name = "rustls-webpki" version = "0.101.7" @@ -2633,17 +2967,6 @@ dependencies = [ "untrusted", ] -[[package]] -name = "rustls-webpki" -version = "0.102.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", -] - [[package]] name = "ryu" version = "1.0.18" @@ -2675,6 +2998,20 @@ dependencies = [ "untrusted", ] +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der 0.6.1", + "generic-array", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -2717,12 +3054,12 @@ checksum = "1a34ad8e4a86884ab42e9b8690e9343abdcfe5fa38a0318cfe1565ba9ad437b4" dependencies = [ "either", "flate2", - "hyper 0.14.30", + "hyper", "indicatif", "log", "quick-xml 0.23.1", "regex", - "reqwest 0.11.27", + "reqwest", "self-replace", "semver", "serde_json", @@ -2758,7 +3095,7 @@ checksum = "6aea2634c86b0e8ef2cfdc0c340baede54ec27b1e46febd7f80dffb2aa44a00e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn", ] [[package]] @@ -2839,6 +3176,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.8" @@ -2912,6 +3260,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "signature" version = "2.2.0" @@ -2980,6 +3338,16 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der 0.6.1", +] + [[package]] name = "spki" version = "0.7.3" @@ -2987,7 +3355,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.9", ] [[package]] @@ -3054,17 +3422,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f3fece30b2dc06d65ecbca97b602db15bf75f932711d60cc604534f1f8b7a03" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.74" @@ -3082,12 +3439,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" -[[package]] -name = "sync_wrapper" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" - [[package]] name = "system-configuration" version = "0.5.1" @@ -3174,15 +3525,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - [[package]] name = "terminal_size" version = "0.3.0" @@ -3227,7 +3569,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn", ] [[package]] @@ -3240,6 +3582,36 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" + +[[package]] +name = "time-macros" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -3265,7 +3637,7 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot 0.12.3", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -3281,7 +3653,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn", ] [[package]] @@ -3300,18 +3672,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.12", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" -dependencies = [ - "rustls 0.23.12", - "rustls-pki-types", + "rustls", "tokio", ] @@ -3374,27 +3735,6 @@ dependencies = [ "winnow 0.6.18", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - [[package]] name = "tower-service" version = "0.3.3" @@ -3420,7 +3760,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn", ] [[package]] @@ -3578,6 +3918,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "vte" version = "0.11.1" @@ -3644,7 +3990,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.74", + "syn", "wasm-bindgen-shared", ] @@ -3678,7 +4024,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3689,21 +4035,6 @@ version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" -[[package]] -name = "wasm-timer" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" -dependencies = [ - "futures", - "js-sys", - "parking_lot 0.11.2", - "pin-utils", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "web-sys" version = "0.3.70" @@ -3746,15 +4077,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -[[package]] -name = "winapi-util" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" -dependencies = [ - "windows-sys 0.59.0", -] - [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -4013,16 +4335,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "winreg" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "xattr" version = "1.3.1" @@ -4034,6 +4346,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "xxhash-rust" version = "0.8.12" @@ -4078,7 +4396,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9d22aba..039cd1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,17 +11,14 @@ rust-version = "1.75" [dependencies] anyhow = "1.0.86" async-trait = "0.1.81" +aws-config = "1.0.1" +aws-sdk-s3 = { version = "1.4.0", features = ["rt-tokio"] } chrono = "0.4.38" clap = "4.5.15" -cmd_lib = "1.9.4" -futures = "0.3.30" lazy_static = "1.5.0" maplit = "1.0.2" rand = "0.8.5" regex = "1.10.6" -reqwest = { version = "0.12.5", features = ["json"] } -reqwest-middleware = "0.3.3" -reqwest-retry = "0.6.1" serde = { version = "1.0.206", features = ["derive"] } serde_json = "1.0.124" serde_yaml = "0.9.33" diff --git a/README.md b/README.md index 3da1417..0812dc6 100644 --- a/README.md +++ b/README.md @@ -112,8 +112,8 @@ Options: Generate burn_in related tasks --burn-in-tests-command Command to invoke burn_in_tests [default: "python buildscripts/burn_in_tests.py run"] - --s3-test-stats-endpoint - S3 endpoint to get test stats from [default: https://mongo-test-stats.s3.amazonaws.com] + --s3-test-stats-bucket + S3 bucket to get test stats from [default: mongo-test-stats] -h, --help Print help ``` diff --git a/docs/generating_tasks.md b/docs/generating_tasks.md index b20b46e..265da58 100644 --- a/docs/generating_tasks.md +++ b/docs/generating_tasks.md @@ -73,7 +73,7 @@ test suites into sub-suites that can be run in parallel on different hosts. For tasks appropriately marked, the `mongo-task-generator` will query the [runtime stats](https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Evergreen-Data-for-Analytics#evergreen-test-statistics) -endpoint https://mongo-test-stats.s3.amazonaws.com/{evg-project-name}/{variant-name}/{task-name} +bucket https://mongo-test-stats.s3.amazonaws.com/{evg-project-name}/{variant-name}/{task-name} and use those stats to divide up the tests into sub-suite with roughly even runtimes. It will then generate "sub-tasks" for each of the "sub-suites" to actually run the tests. @@ -339,8 +339,8 @@ Options: Generate burn_in related tasks --burn-in-tests-command Command to invoke burn_in_tests [default: "python buildscripts/burn_in_tests.py run"] - --s3-test-stats-endpoint - S3 endpoint to get test stats from [default: https://mongo-test-stats.s3.amazonaws.com] + --s3-test-stats-bucket + S3 bucket to get test stats from [default: mongo-test-stats] -h, --help Print help ``` diff --git a/src/evergreen/evg_task_history.rs b/src/evergreen/evg_task_history.rs index 01f8a63..7409295 100644 --- a/src/evergreen/evg_task_history.rs +++ b/src/evergreen/evg_task_history.rs @@ -1,17 +1,12 @@ //! Lookup the history of evergreen tasks. - use anyhow::{bail, Result}; use async_trait::async_trait; -use reqwest::Error; -use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; -use reqwest_retry::policies::ExponentialBackoff; -use reqwest_retry::RetryTransientMiddleware; use serde::Deserialize; use std::collections::HashMap; use std::fmt::{Display, Formatter}; -use std::time::Duration; +use std::str; +use std::sync::Arc; -const REQWEST_CLIENT_MAX_RETRY_COUNT: u32 = 3; const HOOK_DELIMITER: char = ':'; /// Test stats stored on S3 bucket. @@ -19,14 +14,9 @@ const HOOK_DELIMITER: char = ':'; pub struct S3TestStats { /// Name of test. pub test_name: String, - /// Number of passed tests. - pub num_pass: u64, - /// Number of failed tests. - pub num_fail: u64, /// Average duration of passed tests. pub avg_duration_pass: f64, } - /// Runtime information of hooks that ran in evergreen. #[derive(Debug, Clone)] pub struct HookRuntimeHistory { @@ -37,7 +27,6 @@ pub struct HookRuntimeHistory { /// Average runtime of hook. pub average_runtime: f64, } - impl Display for HookRuntimeHistory { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( @@ -47,7 +36,6 @@ impl Display for HookRuntimeHistory { ) } } - /// Runtime history of a test in evergreen. #[derive(Debug, Clone)] pub struct TestRuntimeHistory { @@ -58,7 +46,6 @@ pub struct TestRuntimeHistory { /// Hooks runtime information of hooks that ran with the test. pub hooks: Vec, } - impl Display for TestRuntimeHistory { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { writeln!(f, "{}: {}", self.test_name, self.average_runtime)?; @@ -68,7 +55,6 @@ impl Display for TestRuntimeHistory { Ok(()) } } - /// Runtime history of a task from evergreen. #[derive(Debug, Clone)] pub struct TaskRuntimeHistory { @@ -77,22 +63,9 @@ pub struct TaskRuntimeHistory { /// Map of tests to the runtime history for that test. pub test_map: HashMap, } - /// A service for querying task history from evergreen. #[async_trait] pub trait TaskHistoryService: Send + Sync { - /// Build URL to send request to. - /// - /// # Arguments - /// - /// * `task` - Name of task to query. - /// * `variant` - Name of build variant to query. - /// - /// # Returns - /// - /// URL to send request to. - fn build_url(&self, task: &str, variant: &str) -> String; - /// Get the test runtime history of the given task. /// /// # Arguments @@ -108,34 +81,35 @@ pub trait TaskHistoryService: Send + Sync { /// An implementation of the task history service. pub struct TaskHistoryServiceImpl { - /// Reqwest client. - client: ClientWithMiddleware, - /// S3 endpoint to get test stats from. - s3_test_stats_endpoint: String, + // AWS S3 client, and a sempahore to limit concurrent client usage. + s3_client: aws_sdk_s3::Client, + semaphore: Arc, + /// S3 bucket to get test stats from. + s3_test_stats_bucket: String, /// Evergreen project to query. evg_project: String, } - impl TaskHistoryServiceImpl { /// Create a new instance of the task history service. /// /// # Arguments /// - /// * `client` - Reqwest client. - /// * `s3_test_stats_endpoint` - S3 endpoint to get test stats from. + /// * `s3_client` - AWS s3 client. + /// * `s3_test_stats_bucket` - S3 bucket to get test stats from. /// * `evg_project` - Evergreen project to query. /// /// # Returns /// /// New instance of the task history service implementation. pub fn new( - client: ClientWithMiddleware, - s3_test_stats_endpoint: String, + s3_client: aws_sdk_s3::Client, + s3_test_stats_bucket: String, evg_project: String, ) -> Self { Self { - client, - s3_test_stats_endpoint, + s3_client, + semaphore: Arc::new(tokio::sync::Semaphore::new(20)), + s3_test_stats_bucket, evg_project, } } @@ -143,23 +117,6 @@ impl TaskHistoryServiceImpl { #[async_trait] impl TaskHistoryService for TaskHistoryServiceImpl { - /// Build URL to send request to. - /// - /// # Arguments - /// - /// * `task` - Name of task to query. - /// * `variant` - Name of build variant to query. - /// - /// # Returns - /// - /// URL to send request to. - fn build_url(&self, task: &str, variant: &str) -> String { - format!( - "{}/{}/{}/{}", - self.s3_test_stats_endpoint, self.evg_project, variant, task - ) - } - /// Get the test runtime history of the given task. /// /// # Arguments @@ -171,17 +128,29 @@ impl TaskHistoryService for TaskHistoryServiceImpl { /// /// The runtime history of tests belonging to the given suite on the given build variant. async fn get_task_history(&self, task: &str, variant: &str) -> Result { - let url = self.build_url(task, variant); - let response = self.client.get(url).send().await?; - let stats: Result, Error> = - Ok(response.json::>().await?); + let key = format!("{}/{}/{}", self.evg_project, variant, task); + + // Acquire a permit for concurrent use of the s3 client, limiting too many + // parallel connections that will result in network failures. + let permit = self.semaphore.acquire().await.unwrap(); + + let object = self + .s3_client + .get_object() + .bucket(&self.s3_test_stats_bucket) + .key(key) + .send() + .await?; + let body = &object.body.collect().await?.to_vec(); + let stats = serde_json::from_str::>(str::from_utf8(body)?); + + drop(permit); if let Ok(stats) = stats { // Split the returned stats into stats for hooks and tests. Also attach the hook stats // to the test that they ran with. let hook_map = gather_hook_stats(&stats); let test_map = gather_test_stats(&stats, &hook_map); - Ok(TaskRuntimeHistory { task_name: task.to_string(), test_map, @@ -192,24 +161,6 @@ impl TaskHistoryService for TaskHistoryServiceImpl { } } -/// Build retryable reqwest client. -/// -/// # Returns -/// -/// Retryable reqwest client. -pub fn build_retryable_client() -> ClientWithMiddleware { - let retry_policy = - ExponentialBackoff::builder().build_with_max_retries(REQWEST_CLIENT_MAX_RETRY_COUNT); - ClientBuilder::new( - reqwest::Client::builder() - .timeout(Duration::from_secs(30)) - .build() - .unwrap(), - ) - .with(RetryTransientMiddleware::new_with_policy(retry_policy)) - .build() -} - /// Convert the list of stats into a map of test names to test stats. /// /// Also include hook information for all tests with their stats. @@ -249,10 +200,8 @@ fn gather_test_stats( } } } - test_map } - /// Gather all the hook stats in the given list into a map by the test the hooks ran with. /// /// # Arguments @@ -289,7 +238,6 @@ fn gather_hook_stats(stat_list: &[S3TestStats]) -> HashMap HashMap bool { identifier.contains(HOOK_DELIMITER) } - /// Get the test name part of a given hook identifier. /// /// # Arguments @@ -317,7 +264,6 @@ fn is_hook(identifier: &str) -> bool { fn hook_test_name(identifier: &str) -> &str { identifier.split(HOOK_DELIMITER).next().unwrap() } - /// Get the hook name part of a given hook identifier. /// /// # Arguments @@ -330,7 +276,6 @@ fn hook_test_name(identifier: &str) -> &str { fn hook_hook_name(identifier: &str) -> &str { identifier.split(HOOK_DELIMITER).last().unwrap() } - /// Normalize the given test files. /// /// Converts windows path separators (\) to unix style (/). @@ -345,7 +290,6 @@ fn hook_hook_name(identifier: &str) -> &str { fn normalize_test_file(test_file: &str) -> String { test_file.replace('\\', "/") } - /// Get the base name of the given test file. /// /// # Arguments @@ -359,30 +303,24 @@ pub fn get_test_name(test_file: &str) -> String { let s = test_file.split('/'); s.last().unwrap().trim_end_matches(".js").to_string() } - #[cfg(test)] mod tests { use rstest::rstest; - use super::*; - #[rstest] #[case("some/random/test", false)] #[case("some/random/test:hook1", true)] fn test_is_hook(#[case] hook_name: &str, #[case] expected_is_hook: bool) { assert_eq!(is_hook(hook_name), expected_is_hook); } - #[test] fn test_hook_test_name() { assert_eq!(hook_test_name("my_test:my_hook"), "my_test"); } - #[test] fn test_hook_hook_name() { assert_eq!(hook_hook_name("my_test:my_hook"), "my_hook"); } - // normalize test name tests. #[rstest] #[case("jstests\\core\\add1.js", "jstests/core/add1.js")] @@ -391,10 +329,8 @@ mod tests { #[case("jstests/core/add1", "jstests/core/add1")] fn test_normalize_tests(#[case] test_file: &str, #[case] expected_name: &str) { let normalized_name = normalize_test_file(test_file); - assert_eq!(&normalized_name, expected_name); } - // get_test_name tests. #[rstest] #[case("jstests/core/add1.js", "add1")] diff --git a/src/lib.rs b/src/lib.rs index 73ca4e7..1651781 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,15 +10,19 @@ use std::{ collections::{HashMap, HashSet}, path::{Path, PathBuf}, sync::{Arc, Mutex}, + time::Duration, vec, }; use anyhow::{bail, Result}; use async_trait::async_trait; +use aws_config::{ + meta::region::RegionProviderChain, retry::RetryConfig, timeout::TimeoutConfig, BehaviorVersion, +}; use evergreen::{ evg_config::{EvgConfigService, EvgProjectConfig}, evg_config_utils::{EvgConfigUtils, EvgConfigUtilsImpl}, - evg_task_history::{build_retryable_client, TaskHistoryServiceImpl}, + evg_task_history::TaskHistoryServiceImpl, }; use evergreen_names::{ BURN_IN_TAGS, BURN_IN_TAG_COMPILE_TASK_DEPENDENCY, BURN_IN_TAG_INCLUDE_BUILD_VARIANTS, @@ -143,8 +147,8 @@ pub struct ExecutionConfiguration<'a> { pub include_fully_disabled_feature_tests: bool, /// Command to execute burn_in_tests. pub burn_in_tests_command: &'a str, - /// S3 endpoint to get test stats from. - pub s3_test_stats_endpoint: &'a str, + /// S3 bucket to get test stats from. + pub s3_test_stats_bucket: &'a str, pub subtask_limits: SubtaskLimits, } @@ -184,7 +188,10 @@ impl Dependencies { /// # Returns /// /// A set of dependencies to run against. - pub fn new(execution_config: ExecutionConfiguration) -> Result { + pub fn new( + execution_config: ExecutionConfiguration, + s3_client: aws_sdk_s3::Client, + ) -> Result { let fs_service = Arc::new(FsServiceImpl::new()); let discovery_service = Arc::new(ResmokeProxy::new( execution_config.resmoke_command, @@ -207,10 +214,9 @@ impl Dependencies { execution_config.config_location.to_string(), gen_sub_tasks_config, )); - let client = build_retryable_client(); let task_history_service = Arc::new(TaskHistoryServiceImpl::new( - client, - execution_config.s3_test_stats_endpoint.to_string(), + s3_client, + execution_config.s3_test_stats_bucket.to_string(), execution_config.project_info.evg_project.clone(), )); let resmoke_config_actor = @@ -987,6 +993,27 @@ fn create_burn_in_tasks_worker( }) } +pub async fn build_s3_client() -> aws_sdk_s3::Client { + let region_provider = RegionProviderChain::default_provider().or_else("us-east-1"); + + let timeout_config = TimeoutConfig::builder() + .operation_timeout(Duration::from_secs(120)) + .operation_attempt_timeout(Duration::from_secs(30)) + .read_timeout(Duration::from_secs(15)) + .build(); + + let retry_config = RetryConfig::standard().with_max_attempts(3); + + let config = aws_config::defaults(BehaviorVersion::latest()) + .timeout_config(timeout_config) + .retry_config(retry_config) + .region(region_provider) + .load() + .await; + + aws_sdk_s3::Client::new(&config) +} + #[cfg(test)] mod tests { use rstest::rstest; diff --git a/src/main.rs b/src/main.rs index 782465a..7e6643d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,7 +7,8 @@ use std::{ use anyhow::Result; use clap::Parser; use mongo_task_generator::{ - generate_configuration, Dependencies, ExecutionConfiguration, ProjectInfo, SubtaskLimits, + build_s3_client, generate_configuration, Dependencies, ExecutionConfiguration, ProjectInfo, + SubtaskLimits, }; use serde::Deserialize; use tracing::{error, event, Level}; @@ -18,7 +19,7 @@ const DEFAULT_EVG_PROJECT_FILE: &str = "etc/evergreen.yml"; const DEFAULT_RESMOKE_COMMAND: &str = "python buildscripts/resmoke.py"; const DEFAULT_BURN_IN_TESTS_COMMAND: &str = "python buildscripts/burn_in_tests.py run"; const DEFAULT_TARGET_DIRECTORY: &str = "generated_resmoke_config"; -const DEFAULT_S3_TEST_STATS_ENDPOINT: &str = "https://mongo-test-stats.s3.amazonaws.com"; +const DEFAULT_S3_TEST_STATS_BUCKET: &str = "mongo-test-stats"; const DEFAULT_MAX_SUBTASKS_PER_TASK: &str = "10"; const DEFAULT_DEFAULT_SUBTASKS_PER_TASKS: &str = "5"; const DEFAULT_TEST_RUNTIME_PER_REQUIRED_SUBTASK: &str = "3600"; @@ -130,9 +131,9 @@ struct Args { #[clap(long, default_value = DEFAULT_BURN_IN_TESTS_COMMAND)] burn_in_tests_command: String, - /// S3 endpoint to get test stats from. - #[clap(long, default_value = DEFAULT_S3_TEST_STATS_ENDPOINT)] - s3_test_stats_endpoint: String, + /// S3 bucket to get test stats from. + #[clap(long, default_value = DEFAULT_S3_TEST_STATS_BUCKET)] + s3_test_stats_bucket: String, // Ideal total test runtime (in seconds) for individual subtasks on required // variants, used to determine the number of subtasks for tasks on required variants. @@ -186,7 +187,7 @@ async fn main() { skip_covered_tests: evg_expansions.is_patch && !evg_expansions.run_covered_tests, include_fully_disabled_feature_tests: args.include_fully_disabled_feature_tests, burn_in_tests_command: &args.burn_in_tests_command, - s3_test_stats_endpoint: &args.s3_test_stats_endpoint, + s3_test_stats_bucket: &args.s3_test_stats_bucket, subtask_limits: SubtaskLimits { test_runtime_per_required_subtask: args.test_runtime_per_required_subtask, max_subtasks_per_task: args.max_subtasks_per_task, @@ -194,7 +195,8 @@ async fn main() { large_required_task_runtime_threshold: args.large_required_task_runtime_threshold, }, }; - let deps = Dependencies::new(execution_config).unwrap(); + let s3_client = build_s3_client().await; + let deps = Dependencies::new(execution_config, s3_client).unwrap(); let start = Instant::now(); let result = generate_configuration(&deps, &args.target_directory).await; diff --git a/src/task_types/resmoke_tasks.rs b/src/task_types/resmoke_tasks.rs index acffdb4..a1fbd1e 100644 --- a/src/task_types/resmoke_tasks.rs +++ b/src/task_types/resmoke_tasks.rs @@ -1071,10 +1071,6 @@ mod tests { #[async_trait] impl TaskHistoryService for MockTaskHistoryService { - fn build_url(&self, _task: &str, _variant: &str) -> String { - todo!() - } - async fn get_task_history( &self, _task: &str, From affeb5dd5f02167801fcc172987c8216b1e0bf0e Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Wed, 23 Apr 2025 17:54:01 +0000 Subject: [PATCH 3/6] cargo format --- src/evergreen/evg_task_history.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evergreen/evg_task_history.rs b/src/evergreen/evg_task_history.rs index 7409295..b6173d9 100644 --- a/src/evergreen/evg_task_history.rs +++ b/src/evergreen/evg_task_history.rs @@ -305,8 +305,8 @@ pub fn get_test_name(test_file: &str) -> String { } #[cfg(test)] mod tests { - use rstest::rstest; use super::*; + use rstest::rstest; #[rstest] #[case("some/random/test", false)] #[case("some/random/test:hook1", true)] From 22380156be66cd76db988948ce1d8af4f345bd6e Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Wed, 23 Apr 2025 17:56:20 +0000 Subject: [PATCH 4/6] increment release --- CHANGELOG.md | 4 ++++ Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5c71a3..4d20372 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +## 2.0.0 - 2025-04-23 +* Allow splitting large tasks on required variants based on total test runtime +* Use the AWS-sdk for accessing the s3 bucket of test statistics + ## 1.0.0 - 2025-04-16 * Build, test, and release on Ubuntu 24.04 diff --git a/Cargo.lock b/Cargo.lock index 7091eb8..8f2be49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2070,7 +2070,7 @@ dependencies = [ [[package]] name = "mongo-task-generator" -version = "1.1.0" +version = "2.0.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 039cd1a..71f4f6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "mongo-task-generator" description = "Dynamically split evergreen tasks into subtasks for testing the 10gen/mongo project." license = "Apache-2.0" -version = "1.1.0" +version = "2.0.0" repository = "https://github.com/mongodb/mongo-task-generator" authors = ["DevProd Correctness Team "] edition = "2018" From a6c03e5f8f0458cb7766841621d2a8d7384642d2 Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Wed, 23 Apr 2025 18:05:56 +0000 Subject: [PATCH 5/6] upgrade rust version to 1.81 --- Cargo.toml | 2 +- evergreen.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 71f4f6e..fc6bc4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ version = "2.0.0" repository = "https://github.com/mongodb/mongo-task-generator" authors = ["DevProd Correctness Team "] edition = "2018" -rust-version = "1.75" +rust-version = "1.81" [dependencies] anyhow = "1.0.86" diff --git a/evergreen.yml b/evergreen.yml index b44ea7c..56791e5 100644 --- a/evergreen.yml +++ b/evergreen.yml @@ -85,7 +85,7 @@ pre: - func: cargo_install rust vars: - rust_version: 1.75.0 + rust_version: 1.81.0 - func: cargo_enable nextest From b79230e99a4c6796adf110ca36abf15f4786c523 Mon Sep 17 00:00:00 2001 From: Sean Lyons Date: Wed, 23 Apr 2025 18:26:42 +0000 Subject: [PATCH 6/6] quite a few lint fixes --- src/evergreen/evg_task_history.rs | 3 ++- src/lib.rs | 38 ------------------------------- src/resmoke/resmoke_suite.rs | 1 + src/task_types/burn_in_tests.rs | 2 +- src/task_types/fuzzer_tasks.rs | 1 + src/task_types/multiversion.rs | 2 +- src/task_types/resmoke_tasks.rs | 2 ++ 7 files changed, 8 insertions(+), 41 deletions(-) diff --git a/src/evergreen/evg_task_history.rs b/src/evergreen/evg_task_history.rs index b6173d9..d9f5ab4 100644 --- a/src/evergreen/evg_task_history.rs +++ b/src/evergreen/evg_task_history.rs @@ -59,6 +59,7 @@ impl Display for TestRuntimeHistory { #[derive(Debug, Clone)] pub struct TaskRuntimeHistory { /// Name of task. + #[allow(dead_code)] pub task_name: String, /// Map of tests to the runtime history for that test. pub test_map: HashMap, @@ -218,7 +219,7 @@ fn gather_hook_stats(stat_list: &[S3TestStats]) -> HashMap Result> { - todo!() - } - } - - struct MockConfigExtractionService {} - impl ConfigExtractionService for MockConfigExtractionService { - fn task_def_to_fuzzer_params( - &self, - _task_def: &EvgTask, - _build_variant: &BuildVariant, - ) -> Result { - todo!() - } - - fn task_def_to_resmoke_params( - &self, - _task_def: &EvgTask, - _is_enterprise: bool, - _build_variant: Option<&BuildVariant>, - _platform: Option, - ) -> Result { - todo!() - } - - fn determine_large_distro( - &self, - _generated_suite: &dyn GeneratedSuite, - _build_variant: &BuildVariant, - ) -> Result> { - todo!() - } - } - struct MockMultiversionService {} impl MultiversionService for MockMultiversionService { fn exclude_tags_for_task(&self, _task_name: &str, _mv_mode: Option) -> String { diff --git a/src/resmoke/resmoke_suite.rs b/src/resmoke/resmoke_suite.rs index 9c90f3a..fd88c26 100644 --- a/src/resmoke/resmoke_suite.rs +++ b/src/resmoke/resmoke_suite.rs @@ -88,6 +88,7 @@ impl FromStr for ResmokeSuiteConfig { } } +#[allow(clippy::to_string_trait_impl)] impl ToString for ResmokeSuiteConfig { /// Convert this resmoke suite configuration to a string. fn to_string(&self) -> String { diff --git a/src/task_types/burn_in_tests.rs b/src/task_types/burn_in_tests.rs index 2e54bf2..f978009 100644 --- a/src/task_types/burn_in_tests.rs +++ b/src/task_types/burn_in_tests.rs @@ -440,7 +440,7 @@ impl BurnInService for BurnInServiceImpl { .lookup_build_variant_expansion(COMPILE_VARIANT, base_build_variant) .unwrap_or_else(|| base_build_variant.name.clone()); - let variant_task_dependencies = vec![ + let variant_task_dependencies = [ TaskDependency { name: compile_task_dependency, variant: Some(compile_variant), diff --git a/src/task_types/fuzzer_tasks.rs b/src/task_types/fuzzer_tasks.rs index 17ba9ba..59e1823 100644 --- a/src/task_types/fuzzer_tasks.rs +++ b/src/task_types/fuzzer_tasks.rs @@ -31,6 +31,7 @@ pub struct FuzzerGenTaskParams { /// Multiversion tasks to generate. pub multiversion_generate_tasks: Option>, /// Name of build variant being generated on. + #[allow(dead_code)] pub variant: String, /// Resmoke suite for generated tests. pub suite: String, diff --git a/src/task_types/multiversion.rs b/src/task_types/multiversion.rs index 8a91626..be1cd36 100644 --- a/src/task_types/multiversion.rs +++ b/src/task_types/multiversion.rs @@ -101,7 +101,7 @@ impl MultiversionService for MultiversionServiceImpl { } else { self.multiversion_config.requires_fcv_tag.clone() }; - let tags = vec![ + let tags = [ MULTIVERSION_INCOMPATIBLE.to_string(), BACKPORT_REQUIRED_TAG.to_string(), task_tag, diff --git a/src/task_types/resmoke_tasks.rs b/src/task_types/resmoke_tasks.rs index a1fbd1e..2a99c39 100644 --- a/src/task_types/resmoke_tasks.rs +++ b/src/task_types/resmoke_tasks.rs @@ -224,12 +224,14 @@ pub struct ResmokeSuiteGenerationInfo { pub task_name: String, /// Name of resmoke suite generated task is based on. + #[allow(dead_code)] pub origin_suite: String, /// List of generated sub-suites comprising task. pub sub_suites: Vec, /// If true, sub-tasks should be generated for the multiversion generate tasks. + #[allow(dead_code)] pub require_multiversion_generate_tasks: bool, }