SERVER-68699 Check that resmoke suite selector config has valid test …

…paths

buildscripts/idl/check_stable_api_commands_have_idl_definitions.py

@@ -184,7 +184,7 @@ def __init__(self):
             self.command = ""
 
     # pylint: disable=protected-access
-    configure_resmoke._update_config_vars(FakeArgs())
+    configure_resmoke._update_config_vars(arg_parser, FakeArgs())
     configure_resmoke._set_logging_config()
 
     # Configure Fixture logging.

buildscripts/idl/gen_all_feature_flag_list.py

@@ -68,13 +68,14 @@ def gen_all_feature_flags(idl_dirs: List[str] = None):
         for idl_path in sorted(lib.list_idls(idl_dir)):
             if is_third_party_idl(idl_path):
                 continue
-            doc = parser.parse_file(open(idl_path), idl_path)
+            with open(idl_path) as idl_file:
+                doc = parser.parse_file(idl_file, idl_path)
             for feature_flag in doc.spec.feature_flags:
                 if feature_flag.default.literal != "true":
                     all_flags.append(feature_flag.name)
 
-    force_disabled_flags = yaml.safe_load(
-        open("buildscripts/resmokeconfig/fully_disabled_feature_flags.yml"))
+    with open("buildscripts/resmokeconfig/fully_disabled_feature_flags.yml") as fully_disabled_ffs:
+        force_disabled_flags = yaml.safe_load(fully_disabled_ffs)
 
     return list(set(all_flags) - set(force_disabled_flags))
 

buildscripts/resmokeconfig/matrix_suites/generated_suites/change_streams_pre_images_replica_sets_kill_secondary_jscore_passthrough.yml

@@ -47,17 +47,8 @@ executor:
 matrix_suite: true
 selector:
   exclude_files:
-  - jstests/core/**/geo_update_btree.js
   - jstests/core/**/max_doc_size.js
-  - jstests/core/**/mr_bigobject.js
-  - jstests/core/**/capped_large_docs.js
-  - jstests/core/**/capped_resize.js
   - jstests/core/**/exhaust.js
-  - jstests/core/**/explain_execution_error.js
-  - jstests/core/**/sortb.js
-  - jstests/core/**/sortg.js
-  - jstests/core/**/sortj.js
-  - jstests/core/**/column_store_index_compression.js
   exclude_with_any_tags:
   - assumes_standalone_mongod
   - requires_emptycapped

buildscripts/resmokeconfig/matrix_suites/generated_suites/multiversion_auth_future_git_tag.yml

@@ -26,7 +26,6 @@ matrix_suite: true
 selector:
   exclude_files:
   - jstests/multiVersion/libs/*.js
-  - jstests/multiVersion/targetedTestsLastContinuousFeatures/*.js
   - jstests/multiVersion/genericBinVersion/minor_version_tags_new_old_new.js
   - jstests/multiVersion/genericSetFCVUsage/crash_mongos_against_upgraded_cluster.js
   - jstests/multiVersion/genericBinVersion/load_keys_on_upgrade.js

buildscripts/resmokeconfig/matrix_suites/generated_suites/multiversion_future_git_tag.yml

@@ -20,7 +20,6 @@ matrix_suite: true
 selector:
   exclude_files:
   - jstests/multiVersion/libs/*.js
-  - jstests/multiVersion/targetedTestsLastContinuousFeatures/*.js
   - jstests/multiVersion/genericBinVersion/minor_version_tags_new_old_new.js
   - jstests/multiVersion/genericSetFCVUsage/crash_mongos_against_upgraded_cluster.js
   exclude_with_any_tags:

buildscripts/resmokeconfig/matrix_suites/generated_suites/sharding_auth_audit.yml

@@ -31,7 +31,6 @@ selector:
   - jstests/sharding/*[aA]uth*.js
   - jstests/sharding/analyze_shard_key/*[aA]uth*.js
   - jstests/sharding/query/*[aA]uth*.js
-  - jstests/sharding/change_streams/*[aA]uth*.js
   - jstests/sharding/advance_cluster_time_action_type.js
   - jstests/sharding/query/aggregation_currentop.js
   - jstests/sharding/internal_txns/internal_client_restrictions.js

buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml

@@ -74,9 +74,6 @@
         # Do not execute files with helper functions.
         - jstests/multiVersion/libs/*.js
 
-        # Exclude last-continuous specific tests
-        - jstests/multiVersion/targetedTestsLastContinuousFeatures/*.js
-
         # TODO: SERVER-28104
         - jstests/multiVersion/genericBinVersion/minor_version_tags_new_old_new.js
 
@@ -93,9 +90,6 @@
         # Do not execute files with helper functions.
         - jstests/multiVersion/libs/*.js
 
-        # Exclude last-continuous specific tests
-        - jstests/multiVersion/targetedTestsLastContinuousFeatures/*.js
-
         # TODO: SERVER-28104
         - jstests/multiVersion/genericBinVersion/minor_version_tags_new_old_new.js
 

buildscripts/resmokeconfig/suites/aggregation_auth.yml

@@ -11,8 +11,6 @@ selector:
   exclude_files:
   - jstests/aggregation/extras/*.js
   - jstests/aggregation/data/*.js
-  # Skip any tests that run with auth explicitly.
-  - jstests/aggregation/*[aA]uth*.js
 
 executor:
   config:

buildscripts/resmokeconfig/suites/aggregation_ese.yml

@@ -10,8 +10,6 @@ selector:
   exclude_files:
   - jstests/aggregation/extras/*.js
   - jstests/aggregation/data/*.js
-  # Skip any tests that run with auth explicitly.
-  - jstests/aggregation/*[aA]uth*.js
   exclude_with_any_tags:
   - does_not_support_encrypted_storage_engine
 executor:

buildscripts/resmokeconfig/suites/aggregation_ese_gcm.yml

@@ -10,8 +10,6 @@ selector:
   exclude_files:
   - jstests/aggregation/extras/*.js
   - jstests/aggregation/data/*.js
-  # Skip any tests that run with auth explicitly.
-  - jstests/aggregation/*[aA]uth*.js
   exclude_with_any_tags:
   - does_not_support_encrypted_storage_engine
 executor:

buildscripts/resmokeconfig/suites/failpoints_auth.yml

@@ -8,9 +8,6 @@ test_kind: js_test
 selector:
   roots:
   - jstests/fail_point/*.js
-  exclude_files:
-  # Skip any tests that run with auth explicitly.
-  - jstests/fail_point/*[aA]uth*.js
 
 # Failpoint tests start their own mongod's.
 executor:

buildscripts/resmokeconfig/suites/query_golden_classic.yml

@@ -4,7 +4,7 @@ selector:
   roots:
   - jstests/query_golden/**/*.js
   exclude_files:
-  - jstests/query_golden/libs/**
+  - jstests/query_golden/libs/**/*.js
   exclude_with_any_tags:
   - requires_cqf
 executor:

buildscripts/resmokeconfig/suites/query_golden_cqf.yml

@@ -4,7 +4,7 @@ selector:
   roots:
   - jstests/query_golden/**/*.js
   exclude_files:
-  - jstests/query_golden/libs/**
+  - jstests/query_golden/libs/**/*.js
 executor:
   archive:
     hooks:

buildscripts/resmokeconfig/suites/replica_sets_kill_secondaries_jscore_passthrough.yml

@@ -5,26 +5,10 @@ selector:
   - jstests/core/*.js
   - jstests/core/txns/**/*.js
   exclude_files:
-  # The following tests perform a write with a writeConcern of w=2 when 'testingReplication' is
-  # true. This causes the test to hang because the secondary is running with the "rsSyncApplyStop"
-  # failpoint enabled.
-  - jstests/core/**/geo_update_btree.js
   # The following tests create large oplog entries, which can cause the secondary to fall off the
   # primary's oplog when run as a part of burn_in_tests.
   - jstests/core/**/max_doc_size.js
-  - jstests/core/**/mr_bigobject.js
-  - jstests/core/**/capped_large_docs.js
-  - jstests/core/**/capped_resize.js
   - jstests/core/**/exhaust.js
-  # The following tests also create large oplog entries due to the maximum blocking sort size being
-  # 100 MB.
-  - jstests/core/**/explain_execution_error.js
-  - jstests/core/**/sortb.js
-  - jstests/core/**/sortg.js
-  - jstests/core/**/sortj.js
-  # The following test tries to directly query secondaries, which fails if the secondary has been
-  # killed.
-  - jstests/core/**/column_store_index_compression.js
   exclude_with_any_tags:
   - assumes_standalone_mongod
   # emptycapped is not supported with rollback using recover-to-timestamp.

buildscripts/resmokeconfig/suites/sharding_auth.yml

@@ -14,7 +14,6 @@ selector:
   - jstests/sharding/*[aA]uth*.js
   - jstests/sharding/analyze_shard_key/*[aA]uth*.js
   - jstests/sharding/query/*[aA]uth*.js
-  - jstests/sharding/change_streams/*[aA]uth*.js
 
   - jstests/sharding/advance_cluster_time_action_type.js
   - jstests/sharding/query/aggregation_currentop.js

buildscripts/resmokeconfig/suites/sharding_csrs_continuous_config_stepdown.yml

@@ -10,7 +10,6 @@ selector:
   - jstests/sharding/*[aA]uth*.js
   - jstests/sharding/analyze_shard_key/*[aA]uth*.js
   - jstests/sharding/query/*[aA]uth*.js
-  - jstests/sharding/change_streams/*[aA]uth*.js
   - jstests/sharding/cluster_time_across_add_shard.js
 
   - jstests/sharding/internal_txns/internal_client_restrictions.js

buildscripts/resmokelib/config.py

@@ -151,6 +151,9 @@
     # Config Dir
     "config_dir": "buildscripts/resmokeconfig",
 
+    # Directory with jstests
+    "jstests_dir": "jstests",
+
     # UndoDB options
     "undo_recorder_path": None,
 
@@ -616,6 +619,9 @@ def all_options(cls):
 CONFIG_DIR = None
 LOGGER_DIR = None
 
+# Where to look for jstests existence
+JSTESTS_DIR = None
+
 # Generated logging config for the current invocation.
 LOGGING_CONFIG: dict = {}
 SHORTEN_LOGGER_NAME_CONFIG: dict = {}

buildscripts/resmokelib/configure_resmoke.py

@@ -37,7 +37,7 @@
 def validate_and_update_config(parser, args):
     """Validate inputs and update config module."""
     _validate_options(parser, args)
-    _update_config_vars(args)
+    _update_config_vars(parser, args)
     _update_symbolizer_secrets()
     _validate_config(parser)
     _set_logging_config()
@@ -223,7 +223,7 @@ def _set_up_tracing(
     return success
 
 
-def _update_config_vars(values):
+def _update_config_vars(parser, values):
     """Update the variables of the config module."""
 
     config = _config.DEFAULTS.copy()
@@ -308,8 +308,8 @@ def setup_feature_flags():
     _config.EXCLUDE_WITH_ANY_TAGS.extend(
         utils.default_if_none(_tags_from_list(config.pop("exclude_with_any_tags")), []))
 
-    force_disabled_flags = yaml.safe_load(
-        open("buildscripts/resmokeconfig/fully_disabled_feature_flags.yml"))
+    with open("buildscripts/resmokeconfig/fully_disabled_feature_flags.yml") as fully_disabled_ffs:
+        force_disabled_flags = yaml.safe_load(fully_disabled_ffs)
 
     _config.EXCLUDE_WITH_ANY_TAGS.extend(force_disabled_flags)
 
@@ -537,6 +537,9 @@ def _merge_set_params(param_list):
     # Config Dir options.
     _config.CONFIG_DIR = config.pop("config_dir")
 
+    # Directory with jstests option
+    _config.JSTESTS_DIR = config.pop("jstests_dir")
+
     # Configure evergreen task documentation
     if _config.EVERGREEN_TASK_NAME:
         task_name = utils.get_task_name_without_suffix(_config.EVERGREEN_TASK_NAME,
@@ -560,6 +563,10 @@ def configure_tests(test_files, replay_file):
         # Treat `resmoke run @to_replay` as `resmoke run --replayFile to_replay`
         if len(test_files) == 1 and test_files[0].startswith("@"):
             to_replay = test_files[0][1:]
+        elif len(test_files) > 1 and any(test_file.startswith("@") for test_file in test_files):
+            parser.error(
+                "Cannot use @replay with additional test files listed on the command line invocation."
+            )
         elif replay_file:
             to_replay = replay_file
 

buildscripts/resmokelib/errors.py

@@ -98,3 +98,15 @@ class RequiresForceRemove(ResmokeError):
     """Exception raised when a directory cannot be removed."""
 
     pass
+
+
+class TestExcludedFromSuiteError(ResmokeError):
+    """Exception raised when an excluded from suite test passed as positional argument to resmoke."""
+
+    pass
+
+
+class SuiteSelectorConfigurationError(ResmokeError):
+    """Exceptions raised when selector section in suite configuration has errors."""
+
+    pass

buildscripts/resmokelib/parser.py

@@ -34,6 +34,9 @@ def get_parser(usage=None):
     subparsers = parser.add_subparsers(dest="command")
     parser.add_argument("--configDir", dest="config_dir", metavar="CONFIG_DIR",
                         help="Directory to search for resmoke configuration files")
+    parser.add_argument(
+        "--jstestsDir", dest="jstests_dir", metavar="CONFIG_DIR",
+        help="Directory to search for jstests files existence while suite validation")
 
     # Add sub-commands.
     for plugin in _PLUGINS:

buildscripts/resmokelib/run/__init__.py

@@ -620,7 +620,7 @@ def _get_suites(self) -> List[Suite]:
         except errors.InvalidMatrixSuiteError as err:
             self._resmoke_logger.error("Failed to get matrix suite: %s", str(err))
             self.exit(1)
-        except errors.ResmokeError as err:
+        except errors.TestExcludedFromSuiteError as err:
             self._resmoke_logger.error(
                 "Cannot run excluded test in suite config. Use '--force-excluded-tests' to override: %s",
                 str(err))