From 0e294eefbbd65b7b1fc2967e33a857a477ee0a16 Mon Sep 17 00:00:00 2001 From: Mikhail Shchatko Date: Fri, 26 Aug 2022 16:12:31 +0300 Subject: [PATCH 1/5] DAG-1992 Generate only burn_in tasks when --burn-in is passed --- CHANGELOG.md | 3 +++ Cargo.lock | 2 +- Cargo.toml | 2 +- src/lib.rs | 18 ++++++++++------- src/main.rs | 6 ++++++ src/resmoke/burn_in_proxy.rs | 38 ++++++++++++++++++++++++++---------- tests/integration_test.rs | 33 +++++++++++++++++++++++++++++++ tests/mocks/burn_in_tests.py | 22 +++++++++++++++++++++ 8 files changed, 105 insertions(+), 19 deletions(-) create mode 100644 tests/mocks/burn_in_tests.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a6f7e3c..7c7068e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.6.0 - 2022-08-26 +* Generate only burn_in tasks when --burn-in is passed. + ## 0.5.3 - 2022-08-19 * Distribute tests without historic runtime data evenly between subsuites. diff --git a/Cargo.lock b/Cargo.lock index 6634b2a..cf76c14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1219,7 +1219,7 @@ dependencies = [ [[package]] name = "mongo-task-generator" -version = "0.5.3" +version = "0.6.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 151f619..336f6b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mongo-task-generator" -version = "0.5.3" +version = "0.6.0" repository = "https://github.com/mongodb/mongo-task-generator" authors = ["Decision Automation Group "] edition = "2018" diff --git a/src/lib.rs b/src/lib.rs index 21fc116..aad3d6d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -128,6 +128,8 @@ pub struct ExecutionConfiguration<'a> { pub config_location: &'a str, /// Should burn_in tasks be generated. pub gen_burn_in: bool, + /// Command to execute burn_in_tests. + pub burn_in_tests_command: &'a str, } /// Collection of services needed to execution. @@ -208,7 +210,7 @@ impl Dependencies { execution_config.gen_burn_in, )); - let burn_in_discovery = Arc::new(BurnInProxy::new()); + let burn_in_discovery = Arc::new(BurnInProxy::new(execution_config.burn_in_tests_command)); let burn_in_service = Arc::new(BurnInServiceImpl::new( burn_in_discovery, gen_resmoke_task_service, @@ -467,8 +469,8 @@ impl GenerateTasksService for GenerateTasksServiceImpl { for task in &build_variant.tasks { // Burn in tasks could be different for each build variant, so we will always // handle them. - if task.name == BURN_IN_TESTS { - if self.gen_burn_in { + if self.gen_burn_in { + if task.name == BURN_IN_TESTS { thread_handles.push(create_burn_in_worker( deps, task_map.clone(), @@ -477,11 +479,8 @@ impl GenerateTasksService for GenerateTasksServiceImpl { generated_tasks.clone(), )); } - continue; - } - if task.name == BURN_IN_TAGS { - if self.gen_burn_in { + if task.name == BURN_IN_TAGS { for base_bv_name in self .evg_config_utils .lookup_and_split_by_whitespace_build_variant_expansion( @@ -501,6 +500,11 @@ impl GenerateTasksService for GenerateTasksServiceImpl { )); } } + + continue; + } + + if task.name == BURN_IN_TESTS || task.name == BURN_IN_TAGS { continue; } diff --git a/src/main.rs b/src/main.rs index 13f8662..50a8b31 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,6 +16,7 @@ use tracing_subscriber::fmt::format; const DEFAULT_EVG_AUTH_FILE: &str = "~/.evergreen.yml"; const DEFAULT_EVG_PROJECT_FILE: &str = "etc/evergreen.yml"; const DEFAULT_RESMOKE_COMMAND: &str = "python buildscripts/resmoke.py"; +const DEFAULT_BURN_IN_TESTS_COMMAND: &str = "python buildscripts/burn_in_tests.py"; const DEFAULT_TARGET_DIRECTORY: &str = "generated_resmoke_config"; /// Expansions from evergreen to determine settings for how task should be generated. @@ -84,6 +85,10 @@ struct Args { /// Generate burn_in related tasks. #[clap(long)] burn_in: bool, + + /// Command to invoke burn_in_tests. + #[clap(long, default_value = DEFAULT_BURN_IN_TESTS_COMMAND)] + burn_in_tests_command: String, } /// Configure logging for the command execution. @@ -116,6 +121,7 @@ async fn main() { generating_task: &evg_expansions.task_name, config_location: &evg_expansions.config_location(), gen_burn_in: args.burn_in, + burn_in_tests_command: &args.burn_in_tests_command, }; let deps = Dependencies::new(execution_config).unwrap(); diff --git a/src/resmoke/burn_in_proxy.rs b/src/resmoke/burn_in_proxy.rs index 8a7a6ff..505a5d8 100644 --- a/src/resmoke/burn_in_proxy.rs +++ b/src/resmoke/burn_in_proxy.rs @@ -36,11 +36,27 @@ pub trait BurnInDiscovery: Send + Sync { fn discover_tasks(&self, build_variant: &str) -> Result>; } -pub struct BurnInProxy {} +pub struct BurnInProxy { + /// Primary command to invoke burn_in_tests (usually `python`). + burn_in_tests_cmd: String, + /// Script to invoke burn_in_tests. + burn_in_tests_script: Vec, +} impl BurnInProxy { - pub fn new() -> Self { - BurnInProxy {} + /// Create a new `BurnInProxy` instance. + /// + /// # Arguments + /// + /// * `burn_in_tests_cmd` - Command to invoke resmoke. + pub fn new(burn_in_tests_cmd: &str) -> Self { + let cmd_parts: Vec<_> = burn_in_tests_cmd.split(' ').collect(); + let cmd = cmd_parts[0]; + let script = cmd_parts[1..].iter().map(|s| s.to_string()).collect(); + Self { + burn_in_tests_cmd: cmd.to_string(), + burn_in_tests_script: script, + } } } @@ -55,13 +71,15 @@ impl BurnInDiscovery for BurnInProxy { /// /// A list of tasks/tests that were discovered by burn_in_tests. fn discover_tasks(&self, build_variant: &str) -> Result> { - let cmd = vec![ - "python", - "buildscripts/burn_in_tests.py", - "--build-variant", - build_variant, - "--yaml", - ]; + let mut cmd = vec![self.burn_in_tests_cmd.as_str()]; + cmd.append( + &mut self + .burn_in_tests_script + .iter() + .map(|s| s.as_str()) + .collect(), + ); + cmd.append(&mut vec!["--build-variant", build_variant, "--yaml"]); let start = Instant::now(); let cmd_output = run_command(&cmd)?; diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 697d618..9f03d05 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -30,3 +30,36 @@ fn test_end2end_execution() { let files = std::fs::read_dir(tmp_dir_path).unwrap(); assert_eq!(2647, files.into_iter().collect::>().len()); } + +#[test] +fn test_end2end_burn_in_execution() { + let mut cmd = Command::cargo_bin("mongo-task-generator").unwrap(); + let tmp_dir = TempDir::new("generated_resmoke_config").unwrap(); + + cmd.args(&[ + "--target-directory", + tmp_dir.path().to_str().unwrap(), + "--expansion-file", + "tests/data/sample_expansions.yml", + "--evg-project-file", + "tests/data/evergreen.yml", + "--evg-auth-file", + "tests/data/sample_evergreen_auth.yml", + "--resmoke-command", + "python3 tests/mocks/resmoke.py", + "--use-task-split-fallback", + "--generate-sub-tasks-config", + "tests/data/sample_generate_subtasks_config.yml", + "--burn-in", + "--burn-in-tests-command", + "python3 tests/mocks/burn_in_tests.py", + ]) + .assert() + .success(); + + let tmp_dir_path = tmp_dir.path(); + assert!(tmp_dir_path.exists()); + + let files = std::fs::read_dir(tmp_dir_path).unwrap(); + assert_eq!(1, files.into_iter().collect::>().len()); +} diff --git a/tests/mocks/burn_in_tests.py b/tests/mocks/burn_in_tests.py new file mode 100644 index 0000000..2d94ff1 --- /dev/null +++ b/tests/mocks/burn_in_tests.py @@ -0,0 +1,22 @@ +"""Mock of burn_in_tests.py for testing task generation.""" +def burn_in_discovery(): + print(""" +discovered_tasks: +- task_name: jsCore + test_list: + - tests/data/tests/test_0.js +- task_name: sharding_jscore_passthrough + test_list: + - tests/data/tests/test_0.js +- task_name: replica_sets_jscore_passthrough + test_list: + - tests/data/tests/test_0.js + """) + + +def main(): + burn_in_discovery() + + +if __name__ == '__main__': + main() From 865fd9e5532c032394297cff7984d7b44910745c Mon Sep 17 00:00:00 2001 From: Mikhail Shchatko Date: Tue, 30 Aug 2022 13:45:28 +0300 Subject: [PATCH 2/5] add docs for burn-in --- docs/generating_tasks.md | 54 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/docs/generating_tasks.md b/docs/generating_tasks.md index 92600c1..8164e27 100644 --- a/docs/generating_tasks.md +++ b/docs/generating_tasks.md @@ -157,6 +157,49 @@ of the task definition. When this tag is present, both the extra setup steps and of multiversion sub-tasks will be preformed. In order to only perform the extra setup steps the `"no_version_combinations"` tag should also be included. +### Burn in tests and burn in tags + +Newly added or modified tests might become flaky. In order to avoid that, those tests can be run +continuously multiple times in a row to see, if the results are consistent. This process is called +burn-in. + +`burn_in_tests_gen` task is used to generate burn-in tasks on the same buildvariant the task is +added to. [Example](https://github.com/mongodb/mongo/blob/81c41bdfdc56f05973fae70e80e80919f18f50c9/etc/evergreen_yml_components/definitions.yml#L3252-L3256) +of task configuration: + +```yaml +- <<: *gen_task_template + name: burn_in_tests_gen + tags: [] + commands: + - func: "generate resmoke tasks" +``` + +`burn_in_tags_gen` task is used to generate separate burn-in buildvariants. This way we can add the +task to the required buildvariant, that will generate another required buildvariants on the fly +based on another non-required buildvariants with only necessary burn-in tasks. + +[Example](https://github.com/mongodb/mongo/blob/81c41bdfdc56f05973fae70e80e80919f18f50c9/etc/evergreen_yml_components/definitions.yml#L4317-L4321) +of task configuration: + +```yaml +- <<: *gen_task_template + name: burn_in_tags_gen + tags: [] + commands: + - func: "generate resmoke tasks" +``` + +`burn_in_tag_buildvariants` buildvariant expansion is used to configure base buildvariant names. +Base buildvariant names should be delimited by spaced. [Example](https://github.com/mongodb/mongo/blob/81c41bdfdc56f05973fae70e80e80919f18f50c9/etc/evergreen.yml#L1257) +of `burn_in_tag_buildvariants` buildvariant expansion: + +```yaml +burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem enterprise-rhel-80-64-bit-multiversion +``` + +Burn-in related tasks are generated when `--burn-in` is passed. + ## Working with generated tasks A generated tasks is typically composed of a number of related sub-tasks. Because evergreen does @@ -221,6 +264,11 @@ if the default value does not apply. store your resmokeconfig is a different directory, you can adjust this value: `python buildscripts/resmoke.py --configDir=path/to/resmokeconfig`. * **target-directory**: Directory to write generated configuration to. This defaults to `generated_resmoke_config`. +* **burn-in**: Whether to generate burn_in related tasks. If specified only burn_in tasks will be + generated. +* **burn-in-tests-command**: How to invoke the burn_in_tests command. The burn_in_tests command is + used to discover modified or added tests and the tasks they being run on. It defaults to + `python buildscripts/burn_in_tests.py`. ## Usage help @@ -232,6 +280,10 @@ USAGE: mongo-task-generator [OPTIONS] --expansion-file OPTIONS: + --burn-in + Generate burn_in related tasks + --burn-in-tests-command + Command to invoke burn_in_tests [default: "python buildscripts/burn_in_tests.py"] --evg-auth-file File with information on how to authenticate against the evergreen API [default: ~/.evergreen.yml] @@ -240,7 +292,7 @@ OPTIONS: --expansion-file File containing expansions that impact task generation --generate-sub-tasks-config - File containing configuration for generating sub-tasks + File containing configuration for generating sub-tasks -h, --help Print help information --resmoke-command From 2b2a6fe6acc0685f268e247191ee723b9b3af5a2 Mon Sep 17 00:00:00 2001 From: Mikhail Shchatko Date: Tue, 30 Aug 2022 13:47:55 +0300 Subject: [PATCH 3/5] add comment --- tests/integration_test.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 9f03d05..8c18348 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -61,5 +61,7 @@ fn test_end2end_burn_in_execution() { assert!(tmp_dir_path.exists()); let files = std::fs::read_dir(tmp_dir_path).unwrap(); + // Only one file `evergreen_config.json` should be generated. + // That means non-burn-in tasks are NOT generated. assert_eq!(1, files.into_iter().collect::>().len()); } From 3e7ea8e5a1905d0aedc83d3ffc2a834d0ad07636 Mon Sep 17 00:00:00 2001 From: Mikhail Shchatko Date: Tue, 30 Aug 2022 18:42:32 +0300 Subject: [PATCH 4/5] update docs --- docs/generating_tasks.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/generating_tasks.md b/docs/generating_tasks.md index 8164e27..879c733 100644 --- a/docs/generating_tasks.md +++ b/docs/generating_tasks.md @@ -160,11 +160,11 @@ the `"no_version_combinations"` tag should also be included. ### Burn in tests and burn in tags Newly added or modified tests might become flaky. In order to avoid that, those tests can be run -continuously multiple times in a row to see, if the results are consistent. This process is called +continuously multiple times in a row to see if the results are consistent. This process is called burn-in. `burn_in_tests_gen` task is used to generate burn-in tasks on the same buildvariant the task is -added to. [Example](https://github.com/mongodb/mongo/blob/81c41bdfdc56f05973fae70e80e80919f18f50c9/etc/evergreen_yml_components/definitions.yml#L3252-L3256) +added to. The [example](https://github.com/mongodb/mongo/blob/81c41bdfdc56f05973fae70e80e80919f18f50c9/etc/evergreen_yml_components/definitions.yml#L3252-L3256) of task configuration: ```yaml @@ -175,11 +175,11 @@ of task configuration: - func: "generate resmoke tasks" ``` -`burn_in_tags_gen` task is used to generate separate burn-in buildvariants. This way we can add the -task to the required buildvariant, that will generate another required buildvariants on the fly -based on another non-required buildvariants with only necessary burn-in tasks. +`burn_in_tags_gen` task is used to generate separate burn-in buildvariants. This way we can burn-in +on the requested buildvariant as well as the other, additional buildvariants to ensure there is no +difference between buildvariants. -[Example](https://github.com/mongodb/mongo/blob/81c41bdfdc56f05973fae70e80e80919f18f50c9/etc/evergreen_yml_components/definitions.yml#L4317-L4321) +The [example](https://github.com/mongodb/mongo/blob/81c41bdfdc56f05973fae70e80e80919f18f50c9/etc/evergreen_yml_components/definitions.yml#L4317-L4321) of task configuration: ```yaml @@ -191,7 +191,7 @@ of task configuration: ``` `burn_in_tag_buildvariants` buildvariant expansion is used to configure base buildvariant names. -Base buildvariant names should be delimited by spaced. [Example](https://github.com/mongodb/mongo/blob/81c41bdfdc56f05973fae70e80e80919f18f50c9/etc/evergreen.yml#L1257) +Base buildvariant names should be delimited by spaces. The [example](https://github.com/mongodb/mongo/blob/81c41bdfdc56f05973fae70e80e80919f18f50c9/etc/evergreen.yml#L1257) of `burn_in_tag_buildvariants` buildvariant expansion: ```yaml From 4157eeff54a28e1314b9901fa650cf9c26886763 Mon Sep 17 00:00:00 2001 From: Mikhail Shchatko Date: Tue, 30 Aug 2022 18:43:54 +0300 Subject: [PATCH 5/5] minor update --- docs/generating_tasks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/generating_tasks.md b/docs/generating_tasks.md index 879c733..a0efda0 100644 --- a/docs/generating_tasks.md +++ b/docs/generating_tasks.md @@ -177,7 +177,7 @@ of task configuration: `burn_in_tags_gen` task is used to generate separate burn-in buildvariants. This way we can burn-in on the requested buildvariant as well as the other, additional buildvariants to ensure there is no -difference between buildvariants. +difference between them. The [example](https://github.com/mongodb/mongo/blob/81c41bdfdc56f05973fae70e80e80919f18f50c9/etc/evergreen_yml_components/definitions.yml#L4317-L4321) of task configuration: