[release] Add option to filter test by name prefix

release/ray_release/buildkite/filter.py

@@ -22,28 +22,38 @@ def _unflattened_lookup(lookup: Dict, flat_key: str, delimiter: str = "/") -> An
 def filter_tests(
     test_collection: List[Test],
     frequency: Frequency,
-    test_attr_regex_filters: Optional[Dict[str, str]] = None,
+    test_filters: Optional[Dict[str, str]] = None,
     prefer_smoke_tests: bool = False,
     run_jailed_tests: bool = False,
     run_unstable_tests: bool = False,
 ) -> List[Tuple[Test, bool]]:
-    if test_attr_regex_filters is None:
-        test_attr_regex_filters = {}
+    if test_filters is None:
+        test_filters = {}
 
     tests_to_run = []
     for test in test_collection:
+        attr_mismatch = False
         # Skip kuberay tests for now.
         # TODO: (khluu) Remove this once we start running KubeRay release tests.
         if test.is_kuberay() and get_global_config()["kuberay_disabled"]:
             continue
-        # First, filter by string attributes
-        attr_mismatch = False
-        for attr, regex in test_attr_regex_filters.items():
-            if not re.fullmatch(regex, _unflattened_lookup(test, attr) or ""):
-                attr_mismatch = True
-                break
+
+        # Check if any test attributes match filters
+        if test_filters:
+            for attr, value in test_filters.items():
+                # Only prefix filter doesn't use regex
+                if attr == "prefix":
+                    if not test.get_name().startswith(value):
+                        attr_mismatch = True
+                        break
+                else:  # Match filters using regex
+                    attr_value = _unflattened_lookup(test, attr) or ""
+                    if not re.fullmatch(value, attr_value):
+                        attr_mismatch = True
+                        break
         if attr_mismatch:
             continue
+
         if not run_jailed_tests:
             clone_test = copy.deepcopy(test)
             clone_test.update_from_s3()

release/ray_release/buildkite/settings.py

@@ -63,23 +63,22 @@ def get_priority(priority_str: str) -> Priority:
     return priority_str_to_enum[priority_str]
 
 
-def get_test_attr_regex_filters(filters_str: str) -> Dict[str, str]:
+def get_test_filters(filters_str: str) -> Dict[str, str]:
     if not filters_str:
         return {}
 
-    test_attr_regex_filters = {}
+    test_filters = {}
     for line in filters_str.splitlines():
         line = line.strip()
         if not line:
             continue
         parts = line.split(":", maxsplit=1)
         if len(parts) != 2:
             raise ReleaseTestConfigError(
-                f"Invalid test attr regex filter: {line}. "
-                "Should be of the form attr:regex"
+                f"Invalid test filter: {line}. " "Should be of the form attr:value"
             )
-        test_attr_regex_filters[parts[0]] = parts[1]
-    return test_attr_regex_filters
+        test_filters[parts[0]] = parts[1]
+    return test_filters
 
 
 def split_ray_repo_str(repo_str: str) -> Tuple[str, str]:
@@ -127,7 +126,7 @@ def get_default_settings() -> Dict:
     settings = {
         "frequency": Frequency.ANY,
         "prefer_smoke_tests": False,
-        "test_attr_regex_filters": None,
+        "test_filters": None,
         "ray_test_repo": None,
         "ray_test_branch": None,
         "priority": Priority.DEFAULT,
@@ -158,12 +157,13 @@ def update_settings_from_environment(settings: Dict) -> Dict:
 
     if "TEST_NAME" in os.environ:
         # This is for backward compatibility.
-        settings["test_attr_regex_filters"] = get_test_attr_regex_filters(
-            "name:" + os.environ["TEST_NAME"]
-        )
+        settings["test_filters"] = get_test_filters("name:" + os.environ["TEST_NAME"])
+
+    if "TEST_FILTERS" in os.environ:
+        settings["test_filters"] = os.environ["TEST_FILTERS"]
 
     if "TEST_ATTR_REGEX_FILTERS" in os.environ:
-        settings["test_attr_regex_filters"] = get_test_attr_regex_filters(
+        settings["test_filters"] = get_test_filters(
             os.environ["TEST_ATTR_REGEX_FILTERS"]
         )
 
@@ -191,17 +191,11 @@ def update_settings_from_buildkite(settings: Dict):
 
     test_name_filter = get_buildkite_prompt_value("release-test-name")
     if test_name_filter:
-        settings["test_attr_regex_filters"] = get_test_attr_regex_filters(
-            "name:" + test_name_filter
-        )
+        settings["test_filters"] = get_test_filters("name:" + test_name_filter)
 
-    test_attr_regex_filters = get_buildkite_prompt_value(
-        "release-test-attr-regex-filters"
-    )
-    if test_attr_regex_filters:
-        settings["test_attr_regex_filters"] = get_test_attr_regex_filters(
-            test_attr_regex_filters
-        )
+    test_filters = get_buildkite_prompt_value("release-test-filters")
+    if test_filters:
+        settings["test_filters"] = get_test_filters(test_filters)
 
     test_priority = get_buildkite_prompt_value("release-priority")
     if test_priority:

release/ray_release/scripts/build_pipeline.py

@@ -79,14 +79,14 @@ def main(
     env = {}
     frequency = settings["frequency"]
     prefer_smoke_tests = settings["prefer_smoke_tests"]
-    test_attr_regex_filters = settings["test_attr_regex_filters"]
+    test_filters = settings["test_filters"]
     priority = settings["priority"]
 
     logger.info(
         f"Found the following buildkite pipeline settings:\n\n"
         f"  frequency =               {settings['frequency']}\n"
         f"  prefer_smoke_tests =      {settings['prefer_smoke_tests']}\n"
-        f"  test_attr_regex_filters = {settings['test_attr_regex_filters']}\n"
+        f"  test_filters =            {settings['test_filters']}\n"
         f"  ray_test_repo =           {settings['ray_test_repo']}\n"
         f"  ray_test_branch =         {settings['ray_test_branch']}\n"
         f"  priority =                {settings['priority']}\n"
@@ -111,7 +111,7 @@ def main(
     filtered_tests = filter_tests(
         test_collection,
         frequency=frequency,
-        test_attr_regex_filters=test_attr_regex_filters,
+        test_filters=test_filters,
         prefer_smoke_tests=prefer_smoke_tests,
         run_jailed_tests=run_jailed_tests,
         run_unstable_tests=run_unstable_tests,