diff --git a/CHANGELOG.md b/CHANGELOG.md index 71fd932..959abc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## 3.3.0 - 2025-06-10 +* Generate tasks for suites that run via bazel + ## 3.0.1 - 2025-05-22 * Add multiversion binary selection dependency to burn-in tasks when needed. diff --git a/Cargo.lock b/Cargo.lock index 0554d9e..ed82003 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2070,7 +2070,7 @@ dependencies = [ [[package]] name = "mongo-task-generator" -version = "3.2.0" +version = "3.3.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index d90ea99..7c0e0c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "mongo-task-generator" description = "Dynamically split evergreen tasks into subtasks for testing the 10gen/mongo project." license = "Apache-2.0" -version = "3.2.0" +version = "3.3.0" repository = "https://github.com/mongodb/mongo-task-generator" authors = ["DevProd Correctness Team "] edition = "2018" diff --git a/README.md b/README.md index a0bfed4..d4716a3 100644 --- a/README.md +++ b/README.md @@ -117,14 +117,27 @@ Options: Disable evergreen task-history queries and use task splitting fallback --resmoke-command Command to invoke resmoke [default: "python buildscripts/resmoke.py"] + --include-fully-disabled-feature-tests + If the generator should include tests that are tagged with fully disabled features --generate-sub-tasks-config File containing configuration for generating sub-tasks --burn-in Generate burn_in related tasks --burn-in-tests-command Command to invoke burn_in_tests [default: "python buildscripts/burn_in_tests.py run"] - --s3-test-stats-bucket + --s3-test-stats-bucket S3 bucket to get test stats from [default: mongo-test-stats] + --test-runtime-per-required-subtask + [default: 3600] + --large-required-task-runtime-threshold + [default: 7200] + --default-subtasks-per-task + [default: 5] + --max-subtasks-per-task + [default: 10] + --bazel-suite-configs + YAML file mapping mapping bazel target names of suite configs to their file location location + -h, --help Print help ``` diff --git a/src/evergreen/evg_config_utils.rs b/src/evergreen/evg_config_utils.rs index 298a4ef..42ae848 100644 --- a/src/evergreen/evg_config_utils.rs +++ b/src/evergreen/evg_config_utils.rs @@ -30,6 +30,8 @@ pub struct MultiversionGenerateTaskConfig { pub suite_name: String, /// Old version to run testing against. pub old_version: String, + /// The bazel test target, if it is a bazel-based resmoke task. + pub bazel_target: Option, } pub trait EvgConfigUtils: Sync + Send { @@ -361,8 +363,12 @@ impl EvgConfigUtils for EvgConfigUtilsImpl { let generated_task_name = remove_gen_suffix(&task.name); if let Some(vars) = optional_vars { - if let Some(ParamValue::String(suite_var)) = vars.get("suite") { - suite_var + if let Some(ParamValue::String(suite)) = vars.get("suite") { + if is_bazel_suite(suite) { + get_bazel_suite_name(suite) + } else { + suite + } } else { generated_task_name } @@ -406,11 +412,18 @@ impl EvgConfigUtils for EvgConfigUtilsImpl { get_func_vars_by_name(task, INITIALIZE_MULTIVERSION_TASKS) { let mut multiversion_generate_tasks = vec![]; - for (suite_name, old_version) in multiversion_task_map { + for (suite, old_version) in multiversion_task_map { + let (suite_name, bazel_target) = if is_bazel_suite(suite) { + (get_bazel_suite_name(suite).to_string(), Some(suite.clone())) + } else { + (suite.clone(), None) + }; + if let ParamValue::String(value) = old_version { multiversion_generate_tasks.push(MultiversionGenerateTaskConfig { - suite_name: suite_name.clone(), + suite_name, old_version: value.clone(), + bazel_target, }); } } @@ -844,6 +857,33 @@ fn get_resmoke_vars(task: &EvgTask) -> Option<&HashMap> { return get_func_vars_by_name(task, RUN_RESMOKE_TESTS); } +/// Checks if a Resmoke suite is a bazel target. +/// +/// # Arguments +/// +/// * `suite` - A suite name from Evergreen YAML. +/// +/// # Returns +/// +/// True if the suite looks like a bazel target (e.g. starts with `//`). +pub fn is_bazel_suite(suite: &str) -> bool { + suite.starts_with("//") +} + +/// Get a suite name from a bazel target. +/// +/// # Arguments +/// +/// * `target` - A bazel target. +/// +/// # Returns +/// +/// A useful suite name, just the name of the target without the bazel package prefix. +pub fn get_bazel_suite_name(target: &str) -> &str { + let (_, name) = target.rsplit_once(':').unwrap(); + name +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; @@ -1413,10 +1453,12 @@ mod tests { MultiversionGenerateTaskConfig { suite_name: "mv_suite1_last_continuous".to_string(), old_version: "last-continuous".to_string(), + bazel_target: None, }, MultiversionGenerateTaskConfig { suite_name: "mv_suite1_last_lts".to_string(), old_version: "last-lts".to_string(), + bazel_target: None, }, ]; assert!(multiversion_generate_tasks diff --git a/src/evergreen_names.rs b/src/evergreen_names.rs index 5520cf3..357f327 100644 --- a/src/evergreen_names.rs +++ b/src/evergreen_names.rs @@ -17,6 +17,8 @@ pub const SETUP_JSTESTFUZZ: &str = "setup jstestfuzz"; pub const RUN_FUZZER: &str = "run jstestfuzz"; /// Function to run generated tasks. pub const RUN_GENERATED_TESTS: &str = "run generated tests"; +/// Function to run generated tasks via 'bazel test'. +pub const RUN_GENERATED_TESTS_VIA_BAZEL: &str = "run generated tests via bazel"; // Function for multi-version tests. /// Function to do setup for multi-version testing. @@ -67,6 +69,8 @@ pub const MULTIVERSION_EXCLUDE_TAG: &str = "multiversion_exclude_tags_version"; pub const REQUIRE_MULTIVERSION_SETUP: &str = "require_multiversion_setup"; /// Arguments to pass to resmoke command. pub const RESMOKE_ARGS: &str = "resmoke_args"; +/// Arguments to pass to bazel command. +pub const BAZEL_ARGS: &str = "bazel_args"; /// Name of suite being executed. pub const SUITE_NAME: &str = "suite"; /// Location where generation task configuration is stored in S3. diff --git a/src/lib.rs b/src/lib.rs index e7396ce..27e5ebd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -51,6 +51,8 @@ use tokio::{runtime::Handle, task::JoinHandle, time}; use tracing::{event, Level}; use utils::fs_service::FsServiceImpl; +use crate::resmoke::resmoke_proxy::BazelConfigs; + mod evergreen; mod evergreen_names; mod generate_sub_tasks_config; @@ -150,6 +152,7 @@ pub struct ExecutionConfiguration<'a> { /// S3 bucket to get test stats from. pub s3_test_stats_bucket: &'a str, pub subtask_limits: SubtaskLimits, + pub bazel_suite_configs: Option, } #[derive(Debug, Clone)] @@ -193,10 +196,15 @@ impl Dependencies { s3_client: aws_sdk_s3::Client, ) -> Result { let fs_service = Arc::new(FsServiceImpl::new()); + let bazel_suite_configs = match execution_config.bazel_suite_configs { + Some(path) => BazelConfigs::from_yaml_file(&path).unwrap_or_default(), + None => BazelConfigs::default(), + }; let discovery_service = Arc::new(ResmokeProxy::new( execution_config.resmoke_command, execution_config.skip_covered_tests, execution_config.include_fully_disabled_feature_tests, + bazel_suite_configs, )); let multiversion_service = Arc::new(MultiversionServiceImpl::new( discovery_service.get_multiversion_config()?, @@ -240,6 +248,11 @@ impl Dependencies { fs_service, gen_resmoke_config, execution_config.subtask_limits, + execution_config + .target_directory + .to_str() + .unwrap_or("") + .to_string(), )); let gen_task_service = Arc::new(GenerateTasksServiceImpl::new( evg_config_service, diff --git a/src/main.rs b/src/main.rs index 7e6643d..3a5b3e3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -152,6 +152,10 @@ struct Args { // Maximum number of subtasks that can be generated for tasks #[clap(long, default_value = DEFAULT_MAX_SUBTASKS_PER_TASK)] max_subtasks_per_task: usize, + + /// YAML file mapping mapping bazel target names of suite configs to their file location location + #[clap(long, value_parser)] + bazel_suite_configs: Option, } /// Configure logging for the command execution. @@ -194,6 +198,7 @@ async fn main() { default_subtasks_per_task: args.default_subtasks_per_task, large_required_task_runtime_threshold: args.large_required_task_runtime_threshold, }, + bazel_suite_configs: args.bazel_suite_configs.as_ref().map(|p| expand_path(p)), }; let s3_client = build_s3_client().await; let deps = Dependencies::new(execution_config, s3_client).unwrap(); diff --git a/src/resmoke/resmoke_proxy.rs b/src/resmoke/resmoke_proxy.rs index f25853a..ead6fb3 100644 --- a/src/resmoke/resmoke_proxy.rs +++ b/src/resmoke/resmoke_proxy.rs @@ -4,7 +4,10 @@ use anyhow::Result; use serde::Deserialize; use tracing::{error, event, Level}; +use crate::evergreen::evg_config_utils::is_bazel_suite; + use super::{external_cmd::run_command, resmoke_suite::ResmokeSuiteConfig}; +use std::collections::HashMap; /// Interface for discovering details about test suites. pub trait TestDiscovery: Send + Sync { @@ -45,6 +48,7 @@ pub struct ResmokeProxy { skip_covered_tests: bool, /// True if test discovery should include tests that are tagged with fully disabled features. include_fully_disabled_feature_tests: bool, + bazel_suite_configs: BazelConfigs, } impl ResmokeProxy { @@ -55,10 +59,12 @@ impl ResmokeProxy { /// * `resmoke_cmd` - Command to invoke resmoke. /// * `skip_covered_tests` - Whether the generator should skip tests run in more complex suites. /// * `include_fully_disabled_feature_tests` - If the generator should include tests that are tagged with fully disabled features. + /// * `bazel_suite_configs` - Optional bazel suite configurations. pub fn new( resmoke_cmd: &str, skip_covered_tests: bool, include_fully_disabled_feature_tests: bool, + bazel_suite_configs: BazelConfigs, ) -> Self { let cmd_parts: Vec<_> = resmoke_cmd.split(' ').collect(); let cmd = cmd_parts[0]; @@ -68,10 +74,46 @@ impl ResmokeProxy { resmoke_script: script, skip_covered_tests, include_fully_disabled_feature_tests, + bazel_suite_configs, } } } +#[derive(Debug, Clone, Default)] +pub struct BazelConfigs { + /// Map of bazel resmoke config targets to their generated suite config YAMLs. + configs: HashMap, +} + +impl BazelConfigs { + pub fn from_yaml_file(path: &Path) -> Result { + let contents = std::fs::read_to_string(path)?; + let configs: Result, serde_yaml::Error> = + serde_yaml::from_str(&contents); + if configs.is_err() { + error!( + file = path.display().to_string(), + contents = &contents, + "Failed to parse bazel configs from yaml file", + ); + } + Ok(Self { configs: configs? }) + } + + /// Get the generated suite config for a bazel resmoke target. + /// + /// # Arguments + /// + /// * `target` - Bazel resmoke test target, like "//buildscripts/resmoke:core". + /// + /// # Returns + /// + /// The path the the generated suite config YAML, like "bazel-out/buildscripts/resmoke/core_config.yml". + pub fn get(&self, target: &str) -> &str { + self.configs.get(&format!("{}_config", target)).unwrap() + } +} + /// Details about tests comprising a test suite. #[derive(Debug, Deserialize)] #[allow(dead_code)] @@ -94,9 +136,15 @@ impl TestDiscovery for ResmokeProxy { /// /// A list of tests belonging to given suite. fn discover_tests(&self, suite_name: &str) -> Result> { + let suite_config = if is_bazel_suite(suite_name) { + self.bazel_suite_configs.get(suite_name) + } else { + suite_name + }; + let mut cmd = vec![&*self.resmoke_cmd]; cmd.append(&mut self.resmoke_script.iter().map(|s| s.as_str()).collect()); - cmd.append(&mut vec!["test-discovery", "--suite", suite_name]); + cmd.append(&mut vec!["test-discovery", "--suite", suite_config]); // When running in a patch build, we use the --skipTestsCoveredByMoreComplexSuites // flag to tell Resmoke to exclude any tests in the given suite that will @@ -114,7 +162,7 @@ impl TestDiscovery for ResmokeProxy { event!( Level::INFO, - suite_name, + suite_config, duration_ms = start.elapsed().as_millis() as u64, "Resmoke test discovery finished" ); @@ -146,9 +194,15 @@ impl TestDiscovery for ResmokeProxy { /// /// Resmoke configuration for the given suite. fn get_suite_config(&self, suite_name: &str) -> Result { + let suite_config = if is_bazel_suite(suite_name) { + self.bazel_suite_configs.get(suite_name) + } else { + suite_name + }; + let mut cmd = vec![&*self.resmoke_cmd]; cmd.append(&mut self.resmoke_script.iter().map(|s| s.as_str()).collect()); - cmd.append(&mut vec!["suiteconfig", "--suite", suite_name]); + cmd.append(&mut vec!["suiteconfig", "--suite", suite_config]); let cmd_output = run_command(&cmd).unwrap(); Ok(ResmokeSuiteConfig::from_str(&cmd_output)?) diff --git a/src/services/config_extraction.rs b/src/services/config_extraction.rs index a5a7f96..7b44ba9 100644 --- a/src/services/config_extraction.rs +++ b/src/services/config_extraction.rs @@ -6,7 +6,7 @@ use shrub_rs::models::{task::EvgTask, variant::BuildVariant}; use crate::{ evergreen::evg_config_utils::EvgConfigUtils, evergreen_names::{ - CONTINUE_ON_FAILURE, FUZZER_PARAMETERS, IDLE_TIMEOUT, LARGE_DISTRO_EXPANSION, + BAZEL_ARGS, CONTINUE_ON_FAILURE, FUZZER_PARAMETERS, IDLE_TIMEOUT, LARGE_DISTRO_EXPANSION, LAST_VERSIONS_EXPANSION, MULTIVERSION, MULTIVERSION_BINARY_SELECTION, NO_MULTIVERSION_GENERATE_TASKS, NPM_COMMAND, NUM_FUZZER_FILES, NUM_FUZZER_TASKS, REPEAT_SUITES, RESMOKE_ARGS, RESMOKE_JOBS_MAX, SHOULD_SHUFFLE_TESTS, @@ -184,6 +184,13 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl { .lookup_build_variant_expansion(UNIQUE_GEN_SUFFIX_EXPANSION, build_variant); let suite = evg_config_utils.find_suite_name(task_def).to_string(); + + let bazel_target = self + .evg_config_utils + .get_gen_task_var(task_def, "suite") + .filter(|s| s.starts_with("//")) + .map(|s| s.to_string()); + Ok(FuzzerGenTaskParams { task_name, variant: build_variant.name.to_string(), @@ -220,6 +227,11 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl { is_enterprise, platform: Some(evg_config_utils.infer_build_variant_platform(build_variant)), gen_task_suffix, + bazel_target, + bazel_args: self + .evg_config_utils + .get_gen_task_var(task_def, BAZEL_ARGS) + .map(|s| s.to_string()), }) } @@ -261,6 +273,12 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl { .get_gen_task_var(task_def, "num_tasks") .map(|str| str.parse().unwrap()); + let bazel_target = self + .evg_config_utils + .get_gen_task_var(task_def, "suite") + .filter(|s| s.starts_with("//")) + .map(|s| s.to_string()); + Ok(ResmokeGenParams { task_name, suite_name: suite, @@ -285,6 +303,10 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl { RESMOKE_ARGS, "", ), + bazel_args: self + .evg_config_utils + .get_gen_task_var(task_def, BAZEL_ARGS) + .map(|s| s.to_string()), resmoke_jobs_max: self .evg_config_utils .lookup_optional_param_u64(task_def, RESMOKE_JOBS_MAX)?, @@ -302,6 +324,7 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl { platform, gen_task_suffix, num_tasks, + bazel_target, }) } diff --git a/src/task_types/burn_in_tests.rs b/src/task_types/burn_in_tests.rs index 613a7b4..616424e 100644 --- a/src/task_types/burn_in_tests.rs +++ b/src/task_types/burn_in_tests.rs @@ -315,6 +315,7 @@ impl BurnInServiceImpl { mv_exclude_tags: suite_info.multiversion_tags.clone(), is_enterprise: false, platform: None, + bazel_target: params.bazel_target.clone(), }; self.gen_resmoke_task_service.build_resmoke_sub_task( diff --git a/src/task_types/fuzzer_tasks.rs b/src/task_types/fuzzer_tasks.rs index d3329ee..f29bdd7 100644 --- a/src/task_types/fuzzer_tasks.rs +++ b/src/task_types/fuzzer_tasks.rs @@ -15,9 +15,10 @@ use crate::{ ADD_GIT_TAG, CONFIGURE_EVG_API_CREDS, CONTINUE_ON_FAILURE, DO_MULTIVERSION_SETUP, DO_SETUP, FUZZER_PARAMETERS, GEN_TASK_CONFIG_LOCATION, GET_PROJECT_WITH_NO_MODULES, IDLE_TIMEOUT, MULTIVERSION_EXCLUDE_TAGS, NPM_COMMAND, REQUIRE_MULTIVERSION_SETUP, RESMOKE_ARGS, - RESMOKE_JOBS_MAX, RUN_FUZZER, RUN_GENERATED_TESTS, SETUP_JSTESTFUZZ, SHOULD_SHUFFLE_TESTS, - SUITE_NAME, TASK_NAME, + RESMOKE_JOBS_MAX, RUN_FUZZER, RUN_GENERATED_TESTS, RUN_GENERATED_TESTS_VIA_BAZEL, + SETUP_JSTESTFUZZ, SHOULD_SHUFFLE_TESTS, SUITE_NAME, TASK_NAME, }, + task_types::resmoke_tasks::replace_resmoke_args_with_bazel_args, utils::task_name::name_generated_task, }; @@ -35,12 +36,16 @@ pub struct FuzzerGenTaskParams { pub variant: String, /// Resmoke suite for generated tests. pub suite: String, + /// The bazel test target, if it is a bazel-based resmoke task. + pub bazel_target: Option, /// Number of javascript files fuzzer should generate. pub num_files: String, /// Number of sub-tasks fuzzer should generate. pub num_tasks: u64, /// Arguments to pass to resmoke invocation. pub resmoke_args: String, + /// Arguments that should be passed to bazel. + pub bazel_args: Option, /// NPM command to perform fuzzer execution. pub npm_command: String, /// Arguments to pass to fuzzer invocation. @@ -68,6 +73,15 @@ pub struct FuzzerGenTaskParams { } impl FuzzerGenTaskParams { + fn is_bazel(&self) -> bool { + self.bazel_target.is_some() + || self + .multiversion_generate_tasks + .as_ref() + .map(|tasks| tasks.iter().any(|t| t.bazel_target.is_some())) + .unwrap_or(false) + } + /// Create parameters to send to fuzzer to generate appropriate fuzzer tests. fn build_fuzzer_parameters(&self) -> HashMap { hashmap! { @@ -288,12 +302,36 @@ fn build_fuzzer_sub_task( commands.extend(vec![ fn_call(SETUP_JSTESTFUZZ), fn_call_with_params(RUN_FUZZER, params.build_fuzzer_parameters()), - fn_call_with_params( - RUN_GENERATED_TESTS, - params.build_run_tests_vars(generated_suite_name, old_version), - ), ]); + let mut run_test_vars = params.build_run_tests_vars(generated_suite_name, old_version); + + if params.is_bazel() { + let target: String = if params.is_multiversion() { + params.multiversion_generate_tasks.as_ref().unwrap()[sub_task_index] + .bazel_target + .clone() + .unwrap() + } else { + params.bazel_target.clone().unwrap() + }; + + run_test_vars.insert("targets".to_string(), ParamValue::from(target.as_ref())); + run_test_vars.insert("compiling_for_test".to_string(), ParamValue::from(true)); + + replace_resmoke_args_with_bazel_args( + &mut run_test_vars, + ¶ms.bazel_args.clone().unwrap_or("".to_string()), + ); + + commands.push(fn_call_with_params( + RUN_GENERATED_TESTS_VIA_BAZEL, + run_test_vars, + )); + } else { + commands.push(fn_call_with_params(RUN_GENERATED_TESTS, run_test_vars)); + } + let formatted_name = format!( "{}{}", sub_task_name, @@ -509,4 +547,32 @@ mod tests { "archive_dist_test_debug" ) } + + #[test] + fn test_build_bazel_fuzzer_sub_task() { + let display_name = "my_task"; + let sub_task_index = 42; + let params = FuzzerGenTaskParams { + task_name: "some task".to_string(), + dependencies: vec!["archive_dist_test_debug".to_string()], + bazel_target: Some("//my/bazel:target".to_string()), + ..Default::default() + }; + + let sub_task = build_fuzzer_sub_task(display_name, sub_task_index, ¶ms, None, None); + + assert_eq!(sub_task.name, "my_task_42"); + assert!(sub_task.commands.is_some()); + let commands = sub_task.commands.unwrap(); + assert_eq!(get_evg_fn_name(&commands[0]), Some("do setup")); + assert_eq!(get_evg_fn_name(&commands[3]), Some("run jstestfuzz")); + assert_eq!( + get_evg_fn_name(&commands[4]), + Some("run generated tests via bazel") + ); + assert_eq!( + sub_task.depends_on.unwrap()[0].name, + "archive_dist_test_debug" + ) + } } diff --git a/src/task_types/multiversion.rs b/src/task_types/multiversion.rs index be1cd36..0e2c403 100644 --- a/src/task_types/multiversion.rs +++ b/src/task_types/multiversion.rs @@ -154,10 +154,12 @@ mod tests { MultiversionGenerateTaskConfig { suite_name: "suite1".to_string(), old_version: "last_lts".to_string(), + bazel_target: None, }, MultiversionGenerateTaskConfig { suite_name: "suite2".to_string(), old_version: "last_continuous".to_string(), + bazel_target: None, }, ]; let multiversion_service = MultiversionServiceImpl { @@ -181,10 +183,12 @@ mod tests { MultiversionGenerateTaskConfig { suite_name: "suite1".to_string(), old_version: "last_lts".to_string(), + bazel_target: None, }, MultiversionGenerateTaskConfig { suite_name: "suite2".to_string(), old_version: "last_continuous".to_string(), + bazel_target: None, }, ]; let multiversion_service = MultiversionServiceImpl { diff --git a/src/task_types/resmoke_config_writer.rs b/src/task_types/resmoke_config_writer.rs index ad3c843..a75374d 100644 --- a/src/task_types/resmoke_config_writer.rs +++ b/src/task_types/resmoke_config_writer.rs @@ -140,7 +140,12 @@ impl WriteConfigActorImpl { .iter() .filter(|s| s.exclude_test_list.is_none()) .map(|s| { - let origin_config = resmoke_config_cache.get_config(&s.origin_suite)?; + let suite_name = if s.bazel_target.is_some() { + s.bazel_target.clone().unwrap() + } else { + s.origin_suite.clone() + }; + let origin_config = resmoke_config_cache.get_config(&suite_name)?; let config = origin_config.with_new_tests(Some(&s.test_list), None); let filename = format!( diff --git a/src/task_types/resmoke_tasks.rs b/src/task_types/resmoke_tasks.rs index 387bc98..b5a1066 100644 --- a/src/task_types/resmoke_tasks.rs +++ b/src/task_types/resmoke_tasks.rs @@ -30,7 +30,7 @@ use crate::{ ADD_GIT_TAG, CONFIGURE_EVG_API_CREDS, DO_MULTIVERSION_SETUP, DO_SETUP, GEN_TASK_CONFIG_LOCATION, GET_PROJECT_WITH_NO_MODULES, MULTIVERSION_EXCLUDE_TAG, MULTIVERSION_EXCLUDE_TAGS_FILE, REQUIRE_MULTIVERSION_SETUP, RESMOKE_ARGS, RESMOKE_JOBS_MAX, - RUN_GENERATED_TESTS, SUITE_NAME, + RUN_GENERATED_TESTS, RUN_GENERATED_TESTS_VIA_BAZEL, SUITE_NAME, }, resmoke::resmoke_proxy::TestDiscovery, utils::{fs_service::FsService, task_name::name_generated_task}, @@ -52,6 +52,8 @@ pub struct ResmokeGenParams { pub multiversion_generate_tasks: Option>, /// Name of suite being generated. pub suite_name: String, + /// The bazel test target, if it is a bazel-based resmoke task. + pub bazel_target: Option, /// Should the generated tasks run on a 'large' distro. pub use_large_distro: bool, /// Should the generated tasks run on a 'xlarge' distro. @@ -64,6 +66,8 @@ pub struct ResmokeGenParams { pub repeat_suites: Option, /// Arguments that should be passed to resmoke. pub resmoke_args: String, + /// Arguments that should be passed to bazel. + pub bazel_args: Option, /// Number of jobs to limit resmoke to. pub resmoke_jobs_max: Option, /// Location where generated task configuration will be stored in S3. @@ -83,6 +87,15 @@ pub struct ResmokeGenParams { } impl ResmokeGenParams { + fn is_bazel(&self) -> bool { + self.bazel_target.is_some() + || self + .multiversion_generate_tasks + .as_ref() + .map(|tasks| tasks.iter().any(|t| t.bazel_target.is_some())) + .unwrap_or(false) + } + /// Build the vars to send to the tasks in the 'run tests' function. /// /// # Arguments @@ -215,6 +228,9 @@ pub struct SubSuite { /// Platform of build_variant the sub-suite is for. pub platform: Option, + + /// The bazel test target, if it is a bazel-based resmoke task. + pub bazel_target: Option, } /// Information needed to generate resmoke configuration files for the generated task. @@ -347,6 +363,9 @@ pub struct GenResmokeTaskServiceImpl { config: GenResmokeConfig, subtask_limits: SubtaskLimits, + + /// Directory to place generated configuration files. + pub target_directory: String, } impl GenResmokeTaskServiceImpl { @@ -362,6 +381,7 @@ impl GenResmokeTaskServiceImpl { /// # Returns /// /// New instance of GenResmokeTaskService. + #[allow(clippy::too_many_arguments)] pub fn new( task_history_service: Arc, test_discovery: Arc, @@ -370,6 +390,7 @@ impl GenResmokeTaskServiceImpl { fs_service: Arc, config: GenResmokeConfig, subtask_limits: SubtaskLimits, + target_directory: String, ) -> Self { Self { task_history_service, @@ -379,6 +400,7 @@ impl GenResmokeTaskServiceImpl { fs_service, config, subtask_limits, + target_directory, } } } @@ -480,6 +502,7 @@ impl GenResmokeTaskServiceImpl { mv_exclude_tags: multiversion_tags.clone(), is_enterprise: params.is_enterprise, platform: params.platform.clone(), + bazel_target: params.bazel_target.clone(), }); } @@ -500,10 +523,15 @@ impl GenResmokeTaskServiceImpl { params: &ResmokeGenParams, multiversion_name: Option<&str>, ) -> Result> { - let suite_name = multiversion_name.unwrap_or(¶ms.suite_name); + let suite_name = if params.is_bazel() { + params.bazel_target.clone().unwrap() + } else { + multiversion_name.unwrap_or(¶ms.suite_name).to_string() + }; + let mut test_list: Vec = self .test_discovery - .discover_tests(suite_name)? + .discover_tests(&suite_name)? .into_iter() .filter(|s| self.fs_service.file_exists(s)) .collect(); @@ -569,6 +597,7 @@ impl GenResmokeTaskServiceImpl { mv_exclude_tags: multiversion_tags.clone(), is_enterprise: params.is_enterprise, platform: params.platform.clone(), + bazel_target: params.bazel_target.clone(), }); } Ok(sub_suites) @@ -591,9 +620,11 @@ impl GenResmokeTaskServiceImpl { ) -> Result> { let mut mv_sub_suites = vec![]; for multiversion_task in params.multiversion_generate_tasks.as_ref().unwrap() { + let mut params_for_multiversion = params.clone(); + params_for_multiversion.bazel_target = multiversion_task.bazel_target.clone(); let suites = self .create_tasks( - params, + ¶ms_for_multiversion, build_variant, Some(&multiversion_task.suite_name.clone()), Some(multiversion_task.old_version.clone()), @@ -789,22 +820,50 @@ impl GenResmokeTaskService for GenResmokeTaskServiceImpl { params.platform.as_deref(), ); - let run_test_vars = - params.build_run_test_vars(&suite_file, sub_suite, &exclude_tags, suite_override); - let formatted_name = format!( "{}{}", suite_file, params.gen_task_suffix.as_deref().unwrap_or("") ); + + let mut run_test_vars = + params.build_run_test_vars(&suite_file, sub_suite, &exclude_tags, suite_override); + + let run_test_fn_name = if params.is_bazel() { + run_test_vars.insert( + "targets".to_string(), + ParamValue::from(sub_suite.bazel_target.clone().unwrap().as_str()), + ); + run_test_vars.insert("compiling_for_test".to_string(), ParamValue::from(true)); + + replace_resmoke_args_with_bazel_args( + &mut run_test_vars, + [ + params.bazel_args.clone().unwrap_or("".to_string()), + format!( + "--test_arg=--suites={}/{}.yml", + self.target_directory, formatted_name, + ), + ] + .join(" ") + .as_str(), + ); + + RUN_GENERATED_TESTS_VIA_BAZEL + } else { + RUN_GENERATED_TESTS + }; + + let commands = Some(resmoke_commands( + run_test_fn_name, + run_test_vars, + params.require_multiversion_setup, + )); + GeneratedSubTask { evg_task: EvgTask { name: formatted_name, - commands: Some(resmoke_commands( - RUN_GENERATED_TESTS, - run_test_vars, - params.require_multiversion_setup, - )), + commands, depends_on: params.get_dependencies(), ..Default::default() }, @@ -814,6 +873,26 @@ impl GenResmokeTaskService for GenResmokeTaskServiceImpl { } } +/// Replaces a `resmoke_args` ParamValue with an equivalent `bazel_args` ParamValue. +pub fn replace_resmoke_args_with_bazel_args( + run_test_vars: &mut HashMap, + bazel_args: &str, +) { + let converted_args: Vec = run_test_vars + .get("resmoke_args") + .unwrap() + .to_string() + .split_whitespace() + .filter(|s| !s.starts_with("--originSuite")) + .map(|s| format!("--test_arg={}", s)) + .collect(); + run_test_vars.remove("resmoke_args"); + run_test_vars.insert( + "bazel_args".to_string(), + ParamValue::from(format!("{} {}", converted_args.join(" "), bazel_args).as_ref()), + ); +} + /// Create a list of commands to run a resmoke task in evergreen. /// /// # Arguments @@ -1162,6 +1241,7 @@ mod tests { default_subtasks_per_task: 5, max_subtasks_per_task: 10, }, + "generated_resmoke_config".to_string(), ) } @@ -1482,10 +1562,12 @@ mod tests { MultiversionGenerateTaskConfig { suite_name: "suite1_last_lts".to_string(), old_version: "last-lts".to_string(), + bazel_target: None, }, MultiversionGenerateTaskConfig { suite_name: "suite1_last_continuous".to_string(), old_version: "last-continuous".to_string(), + bazel_target: None, }, ]), num_tasks: Some(1), @@ -1730,10 +1812,12 @@ mod tests { MultiversionGenerateTaskConfig { suite_name: "suite1_last_lts".to_string(), old_version: "last-lts".to_string(), + bazel_target: None, }, MultiversionGenerateTaskConfig { suite_name: "suite1_last_continuous".to_string(), old_version: "last-continuous".to_string(), + bazel_target: None, }, ]; let params = ResmokeGenParams { @@ -1863,4 +1947,56 @@ mod tests { let min_idx = get_min_index(&running_runtimes); assert_eq!(min_idx, expected_min_idx); } + + #[tokio::test] + async fn test_generate_bazel_resmoke_tasks() { + let num_tasks = 3; + let test_list: Vec = (0..6) + .into_iter() + .map(|i| format!("test_{}.js", i)) + .collect(); + let task_history = TaskRuntimeHistory { + task_name: "my_task".to_string(), + test_map: hashmap! { + "test_0".to_string() => build_mock_test_runtime("test_0.js", 100.0), + "test_1".to_string() => build_mock_test_runtime("test_1.js", 50.0), + "test_2".to_string() => build_mock_test_runtime("test_2.js", 50.0), + "test_3".to_string() => build_mock_test_runtime("test_3.js", 34.0), + "test_4".to_string() => build_mock_test_runtime("test_4.js", 34.0), + "test_5".to_string() => build_mock_test_runtime("test_5.js", 34.0), + }, + }; + let gen_resmoke_service = build_mocked_service(test_list, task_history.clone()); + let params = ResmokeGenParams { + task_name: "my_task".to_string(), + require_multiversion_generate_tasks: false, + num_tasks: Some(num_tasks), + bazel_target: Some("//some:my_task".to_string()), + ..Default::default() + }; + + let suite = gen_resmoke_service + .generate_resmoke_task( + ¶ms, + &BuildVariant { + display_name: Some("build-variant".to_string()), + ..Default::default() + }, + ) + .await + .unwrap(); + + assert_eq!(suite.display_name(), "my_task".to_string()); + assert_eq!(suite.sub_tasks().len(), num_tasks); + + for sub_task in suite.sub_tasks() { + assert!(sub_task.evg_task.commands.is_some()); + let commands = sub_task.evg_task.commands.unwrap(); + assert_eq!(get_evg_fn_name(&commands[0]), Some("do setup")); + assert_eq!( + get_evg_fn_name(&commands[2]), + Some("run generated tests via bazel") + ); + } + } } diff --git a/tests/data/evergreen.yml b/tests/data/evergreen.yml index 438d70f..81f8a2b 100644 --- a/tests/data/evergreen.yml +++ b/tests/data/evergreen.yml @@ -129,7 +129,6 @@ variables: vars: resmoke_args: --help resmoke_jobs_max: 1 - - func: "send benchmark results" - func: "analyze benchmark results" vars: suite: benchmark_suite @@ -864,20 +863,6 @@ functions: content_type: atext-plain display_name: Pip Requirements - "send benchmark results": - - command: json.send - params: - name: perf - file: src/perf.json - - - command: perf.send - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - bucket: mciuploads - prefix: ${task_id}_${execution} - file: src/cedar_report.json - "analyze benchmark results": - *f_expansions_write - *configure_evergreen_api_credentials @@ -3189,7 +3174,6 @@ tasks: suite: benchmarks exec_timeout_secs: 10800 # 3 hour timeout. resmoke_jobs_max: 1 - - func: "send benchmark results" - func: "analyze benchmark results" vars: suite: benchmarks @@ -3203,7 +3187,6 @@ tasks: vars: suite: benchmarks_sharding resmoke_jobs_max: 1 - - func: "send benchmark results" - func: "analyze benchmark results" - <<: *benchmark_template @@ -3215,7 +3198,6 @@ tasks: vars: suite: benchmarks_cst resmoke_jobs_max: 1 - - func: "send benchmark results" - func: "analyze benchmark results" - <<: *run_jepsen_template @@ -4062,17 +4044,17 @@ tasks: name: multiversion_sanity_check_gen tags: ["multiversion", "multiversion_sanity_check"] commands: - - func: "initialize multiversion tasks" - vars: - multiversion_sanity_check_last_continuous_new_new_old: last_continuous - multiversion_sanity_check_last_continuous_new_old_new: last_continuous - multiversion_sanity_check_last_continuous_old_new_new: last_continuous - multiversion_sanity_check_last_lts_new_new_old: last_lts - multiversion_sanity_check_last_lts_new_old_new: last_lts - multiversion_sanity_check_last_lts_old_new_new: last_lts - func: "generate resmoke tasks" vars: run_no_feature_flag_tests: "true" + - func: "initialize multiversion tasks" + vars: + //buildscripts/resmokeconfig:multiversion_sanity_check_last_continuous_new_new_old: last_continuous + //buildscripts/resmokeconfig:multiversion_sanity_check_last_continuous_new_old_new: last_continuous + //buildscripts/resmokeconfig:multiversion_sanity_check_last_continuous_old_new_new: last_continuous + //buildscripts/resmokeconfig:multiversion_sanity_check_last_lts_new_new_old: last_lts + //buildscripts/resmokeconfig:multiversion_sanity_check_last_lts_new_old_new: last_lts + //buildscripts/resmokeconfig:multiversion_sanity_check_last_lts_old_new_new: last_lts - <<: *gen_task_template name: replica_sets_jscore_multiversion_gen diff --git a/tests/data/sample_bazel_suite_configs.yml b/tests/data/sample_bazel_suite_configs.yml new file mode 100644 index 0000000..ea8bcb5 --- /dev/null +++ b/tests/data/sample_bazel_suite_configs.yml @@ -0,0 +1,6 @@ + //buildscripts/resmokeconfig:multiversion_sanity_check_last_continuous_new_new_old_config: bazel-out/multiversion_sanity_check.yml + //buildscripts/resmokeconfig:multiversion_sanity_check_last_continuous_new_old_new_config: bazel-out/multiversion_sanity_check.yml + //buildscripts/resmokeconfig:multiversion_sanity_check_last_continuous_old_new_new_config: bazel-out/multiversion_sanity_check.yml + //buildscripts/resmokeconfig:multiversion_sanity_check_last_lts_new_new_old_config: bazel-out/multiversion_sanity_check.yml + //buildscripts/resmokeconfig:multiversion_sanity_check_last_lts_new_old_new_config: bazel-out/multiversion_sanity_check.yml + //buildscripts/resmokeconfig:multiversion_sanity_check_last_lts_old_new_new_config: bazel-out/multiversion_sanity_check.yml diff --git a/tests/integration_test.rs b/tests/integration_test.rs index a14b38a..9f3f5d8 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -23,6 +23,8 @@ fn test_end2end_execution() { "--use-task-split-fallback", "--generate-sub-tasks-config", "tests/data/sample_generate_subtasks_config.yml", + "--bazel-suite-configs", + "tests/data/sample_bazel_suite_configs.yml", ]) .assert() .success();