Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
SERVER-31470 Move "run tests" logic into evergreen_run_tests.py.
- Loading branch information
Showing
13 changed files
with
534 additions
and
240 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,152 @@ | ||
| #!/usr/bin/env python | ||
|
|
||
| """ | ||
| Command line utility for executing MongoDB tests in Evergreen. | ||
| """ | ||
|
|
||
| from __future__ import absolute_import | ||
|
|
||
| import collections | ||
| import os.path | ||
| import sys | ||
|
|
||
| # Get relative imports to work when the package is not installed on the PYTHONPATH. | ||
| if __name__ == "__main__" and __package__ is None: | ||
| sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | ||
|
|
||
| from buildscripts import resmoke | ||
| from buildscripts import resmokelib | ||
|
|
||
|
|
||
| _TagInfo = collections.namedtuple("_TagInfo", ["tag_name", "evergreen_aware", "suite_options"]) | ||
|
|
||
|
|
||
| class Main(resmoke.Main): | ||
| """ | ||
| A class for executing potentially multiple resmoke.py test suites in a way that handles | ||
| additional options for running unreliable tests in Evergreen. | ||
| """ | ||
|
|
||
| UNRELIABLE_TAG = _TagInfo(tag_name="unreliable", | ||
| evergreen_aware=True, | ||
| suite_options=resmokelib.config.SuiteOptions.ALL_INHERITED._replace( | ||
| report_failure_status="silentfail")) | ||
|
|
||
| RESOURCE_INTENSIVE_TAG = _TagInfo( | ||
| tag_name="resource_intensive", | ||
| evergreen_aware=False, | ||
| suite_options=resmokelib.config.SuiteOptions.ALL_INHERITED._replace(num_jobs=1)) | ||
|
|
||
| RETRY_ON_FAILURE_TAG = _TagInfo( | ||
| tag_name="retry_on_failure", | ||
| evergreen_aware=True, | ||
| suite_options=resmokelib.config.SuiteOptions.ALL_INHERITED._replace( | ||
| fail_fast=False, | ||
| num_repeats=2, | ||
| report_failure_status="silentfail")) | ||
|
|
||
| def _make_evergreen_aware_tags(self, tag_name): | ||
| """ | ||
| Returns a list of resmoke.py tags for task, variant, and distro combinations in Evergreen. | ||
| """ | ||
|
|
||
| tags_format = ["{tag_name}"] | ||
|
|
||
| if resmokelib.config.EVERGREEN_TASK_NAME is not None: | ||
| tags_format.append("{tag_name}|{task_name}") | ||
|
|
||
| if resmokelib.config.EVERGREEN_VARIANT_NAME is not None: | ||
| tags_format.append("{tag_name}|{task_name}|{variant_name}") | ||
|
|
||
| if resmokelib.config.EVERGREEN_DISTRO_ID is not None: | ||
| tags_format.append("{tag_name}|{task_name}|{variant_name}|{distro_id}") | ||
|
|
||
| return [tag.format(tag_name=tag_name, | ||
| task_name=resmokelib.config.EVERGREEN_TASK_NAME, | ||
| variant_name=resmokelib.config.EVERGREEN_VARIANT_NAME, | ||
| distro_id=resmokelib.config.EVERGREEN_DISTRO_ID) | ||
| for tag in tags_format] | ||
|
|
||
| @classmethod | ||
| def _make_tag_combinations(cls): | ||
| """ | ||
| Returns a list of (tag, enabled) pairs representing all possible combinations of all | ||
| possible pairings of whether the tags are enabled or disabled together. | ||
| """ | ||
|
|
||
| combinations = [] | ||
|
|
||
| if resmokelib.config.EVERGREEN_PATCH_BUILD: | ||
| combinations.append(( | ||
| "unreliable and resource intensive", | ||
| ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, True)))) | ||
| combinations.append(( | ||
| "unreliable and not resource intensive", | ||
| ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, False)))) | ||
| combinations.append(( | ||
| "reliable and resource intensive", | ||
| ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, True)))) | ||
| combinations.append(( | ||
| "reliable and not resource intensive", | ||
| ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, False)))) | ||
| else: | ||
| combinations.append(( | ||
| "retry on failure and resource intensive", | ||
| ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, True)))) | ||
| combinations.append(( | ||
| "retry on failure and not resource intensive", | ||
| ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, False)))) | ||
| combinations.append(( | ||
| "run once and resource intensive", | ||
| ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, True)))) | ||
| combinations.append(( | ||
| "run once and not resource intensive", | ||
| ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, False)))) | ||
|
|
||
| return combinations | ||
|
|
||
| def _get_suites(self): | ||
| """ | ||
| Returns a list of resmokelib.testing.suite.Suite instances to execute. | ||
| For every resmokelib.testing.suite.Suite instance returned by resmoke.Main._get_suites(), | ||
| multiple copies of that test suite are run using different resmokelib.config.SuiteOptions() | ||
| depending on whether each tag in the combination is enabled or not. | ||
| """ | ||
|
|
||
| suites = [] | ||
|
|
||
| for suite in resmoke.Main._get_suites(self): | ||
| if suite.test_kind != "js_test": | ||
| # Tags are only support for JavaScript tests, so we leave the test suite alone when | ||
| # running any other kind of test. | ||
| suites.append(suite) | ||
| continue | ||
|
|
||
| for (tag_desc, tag_combo) in self._make_tag_combinations(): | ||
| suite_options_list = [] | ||
|
|
||
| for (tag_info, enabled) in tag_combo: | ||
| if tag_info.evergreen_aware: | ||
| tags = self._make_evergreen_aware_tags(tag_info.tag_name) | ||
| include_tags = {"$anyOf": tags} | ||
| else: | ||
| include_tags = tag_info.tag_name | ||
|
|
||
| if enabled: | ||
| suite_options = tag_info.suite_options._replace(include_tags=include_tags) | ||
| else: | ||
| suite_options = resmokelib.config.SuiteOptions.ALL_INHERITED._replace( | ||
| include_tags={"$not": include_tags}) | ||
|
|
||
| suite_options_list.append(suite_options) | ||
|
|
||
| suite_options = resmokelib.config.SuiteOptions.combine(*suite_options_list) | ||
| suite_options = suite_options._replace(description=tag_desc) | ||
| suites.append(suite.with_options(suite_options)) | ||
|
|
||
| return suites | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| Main().run() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.