[dPWA Testing] Adding parameter wildcards to test generation framework

This change allows critical user journey to specify a wildcard like
"EnumType::All", which will cause the script to expand that single test
into multiple tests for every combination of argument expansion.

This allows us to continue to simplify our test list and implementation.

Bug: b/240571429
Change-Id: I4c45cea08048ab817d246c4208ac428c797e8ff6
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3827079
Commit-Queue: Dibyajyoti Pal <dibyapal@chromium.org>
Reviewed-by: Dibyajyoti Pal <dibyapal@chromium.org>
Auto-Submit: Daniel Murphy <dmurph@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1035189}

chrome/test/webapps/file_reading.py

@@ -19,6 +19,7 @@
 from models import ActionType
 from models import ActionsByName
 from models import CoverageTest
+from models import EnumsByType
 from models import PartialAndFullCoverageByBaseName
 from models import TestIdsByPlatform
 from models import TestIdsByPlatformSet
@@ -67,6 +68,61 @@ def enumerate_all_argument_combinations(argument_types: List[ArgEnum]
     return output
 
 
+def expand_wildcards_in_action(action: str, enums: EnumsByType) -> List[str]:
+    """
+    Takes an action string that could contain enum wildcards, and returns the
+    list of all combinations of actions with all wildcards fully expanded.
+
+    Example input:
+    - action: 'Action(EnumType::All, EnumType::All)'
+    - enums: {'EnumType': EnumType('EnumType', ['Value1', 'Value2'])}
+    Example output:
+    - ['Action(Value1, Value1)', 'Action(Value1, Value2)',
+       'Action(Value2, Value1)', 'Action(Value2, Value2)']
+    """
+    if "::All" not in action:
+        return [action]
+    output: List[str] = []
+    for type, enum in enums.items():
+        wildcard_str = type + "::All"
+        if wildcard_str in action:
+            prefix = action[:action.index(wildcard_str)]
+            postfix = action[action.index(wildcard_str) + len(wildcard_str):]
+            for value in enum.values:
+                output.extend(
+                    expand_wildcards_in_action(prefix + value + postfix,
+                                               enums))
+    return output
+
+
+def expand_tests_from_action_parameter_wildcards(enums: EnumsByType,
+                                                 actions: List[str]
+                                                 ) -> List[List[str]]:
+    """
+    Takes a list of actions for a test that could contain argument wildcards.
+    Returns a list of tests the expand out all combination of argument
+    wildcards.
+    Example input:
+    - actions: ['Action1(EnumType::All), Action2(EnumType::All)']
+    - enums: {'EnumType': EnumType('EnumType', ['Value1', 'Value2'])}
+    Example output:
+    - [['Action1(Value1)', 'Action2(Value1)'],
+       ['Action1(Value1)', 'Action2(Value2)'],
+       ['Action1(Value2)', 'Action2(Value1)'],
+       ['Action1(Value2)', 'Action2(Value2)']]
+    """
+    if not actions:
+        return [[]]
+    current_elements: List[str] = expand_wildcards_in_action(actions[0], enums)
+    output: List[List[str]] = []
+    following_output = expand_tests_from_action_parameter_wildcards(
+        enums, actions[1:])
+    for following_list in following_output:
+        for element in current_elements:
+            output.append([element] + following_list)
+    return output
+
+
 def resolve_bash_style_replacement(output_action_str: str,
                                    argument_values: List[str]):
     for i, arg in enumerate(argument_values):
@@ -159,10 +215,10 @@ def read_platform_supported_actions(csv_file
     return actions_base_name_to_coverage
 
 
-def read_enums_file(enums_file_lines: List[str]) -> Dict[str, ArgEnum]:
+def read_enums_file(enums_file_lines: List[str]) -> EnumsByType:
     """Reads the enums markdown file.
     """
-    enums_by_type: Dict[str, ArgEnum] = {}
+    enums_by_type: EnumsByType = {}
     for i, row in enumerate_markdown_file_lines_to_table_rows(
             enums_file_lines):
         if len(row) < MIN_COLUMNS_ENUMS_FILE:
@@ -321,13 +377,19 @@ def read_actions_file(
             for human_friendly_action_name in output_unresolved_action_names:
                 bash_replaced_name = resolve_bash_style_replacement(
                     human_friendly_action_name, arg_combination)
+
+                # Handle any wildcards in the actions
+                wildcart_expanded_actions = expand_wildcards_in_action(
+                    bash_replaced_name, enums_by_type)
+
                 # Output actions for parameterized actions are not allowed to
                 # use 'defaults', and the action author must explicitly
                 # populate all arguments with bash-style replacements or static
                 # values.
-                output_canonical_action_names.append(
-                    human_friendly_name_to_canonical_action_name(
-                        bash_replaced_name, {}))
+                for action_name in wildcart_expanded_actions:
+                    output_canonical_action_names.append(
+                        human_friendly_name_to_canonical_action_name(
+                            action_name, {}))
 
             if name in actions_by_name:
                 raise ValueError(f"Cannot add duplicate action {name} on row "
@@ -374,6 +436,7 @@ def read_actions_file(
 
 def read_unprocessed_coverage_tests_file(
         coverage_file_lines: List[str], actions_by_name: ActionsByName,
+        enums_by_type: EnumsByType,
         action_base_name_to_default_arg: Dict[str, str]) -> List[CoverageTest]:
     """Reads the coverage tests markdown file.
 
@@ -402,21 +465,30 @@ def read_unprocessed_coverage_tests_file(
         platforms = TestPlatform.get_platforms_from_chars(row[0])
         if len(platforms) == 0:
             raise ValueError(f"Row {i} has invalid platforms: {row[0]}")
-        actions = []
-        for action_name in row[1:]:
-            action_name = action_name.strip()
-            if action_name == "":
-                continue
-            action_name = human_friendly_name_to_canonical_action_name(
-                action_name, action_base_name_to_default_arg)
-            if action_name not in actions_by_name:
-                missing_actions.append(action_name)
-                logging.error(f"Could not find action on row {i!r}: "
-                              f"{action_name}")
-                continue
-            actions.append(actions_by_name[action_name])
-        coverage_test = CoverageTest(actions, platforms)
-        required_coverage_tests.append(coverage_test)
+        # Filter out all blank actions.
+        original_action_strs = [
+            action_str for action_str in row[1:] if action_str.strip()
+        ]
+        # If any of the actions had parameter wildcards (like
+        # "WindowOption::All"), then this expands those into multiple tests.
+        expanded_tests = expand_tests_from_action_parameter_wildcards(
+            enums_by_type, original_action_strs)
+        for test_actions in expanded_tests:
+            actions: List[Action] = []
+            for action_name in test_actions:
+                action_name = action_name.strip()
+                if action_name == "":
+                    continue
+                action_name = human_friendly_name_to_canonical_action_name(
+                    action_name, action_base_name_to_default_arg)
+                if action_name not in actions_by_name:
+                    missing_actions.append(action_name)
+                    logging.error(f"Could not find action on row {i!r}: "
+                                  f"{action_name}")
+                    continue
+                actions.append(actions_by_name[action_name])
+            coverage_test = CoverageTest(actions, platforms)
+            required_coverage_tests.append(coverage_test)
     if missing_actions:
         raise ValueError(f"Actions missing from actions dictionary: "
                          f"{', '.join(missing_actions)}")

chrome/test/webapps/file_reading_unittest.py

@@ -9,6 +9,7 @@
 import unittest
 
 from file_reading import enumerate_all_argument_combinations
+from file_reading import expand_tests_from_action_parameter_wildcards
 from file_reading import enumerate_markdown_file_lines_to_table_rows
 from file_reading import human_friendly_name_to_canonical_action_name
 from file_reading import get_tests_in_browsertest
@@ -173,9 +174,10 @@ def test_coverage_file_reading(self):
         with open(coverage_filename) as f:
             coverage_tsv = f.readlines()
             coverage_tests = read_unprocessed_coverage_tests_file(
-                coverage_tsv, actions, action_base_name_to_default_param)
+                coverage_tsv, actions, enums,
+                action_base_name_to_default_param)
 
-        self.assertEqual(len(coverage_tests), 4)
+        self.assertEqual(6, len(coverage_tests))
 
     def test_browsertest_detection(self):
         browsertest_filename = os.path.join(TEST_DATA_DIR, "tests_default.cc")
@@ -190,6 +192,26 @@ def test_browsertest_detection(self):
                 {TestPlatform.LINUX, TestPlatform.CHROME_OS, TestPlatform.MAC},
                 tests_and_platforms)
 
+    def test_action_param_expansion(self):
+        enum_map: Dict[str, ArgEnum] = {
+            "EnumType": ArgEnum("EnumType", ["Value1", "Value2"], None)
+        }
+        actions: List[str] = [
+            "Action1(EnumType::All)", "Action2(EnumType::All, EnumType::All)"
+        ]
+
+        combinations = expand_tests_from_action_parameter_wildcards(
+            enum_map, actions)
+        expected = [['Action1(Value1)', 'Action2(Value1, Value1)'],
+                    ['Action1(Value2)', 'Action2(Value1, Value1)'],
+                    ['Action1(Value1)', 'Action2(Value1, Value2)'],
+                    ['Action1(Value2)', 'Action2(Value1, Value2)'],
+                    ['Action1(Value1)', 'Action2(Value2, Value1)'],
+                    ['Action1(Value2)', 'Action2(Value2, Value1)'],
+                    ['Action1(Value1)', 'Action2(Value2, Value2)'],
+                    ['Action1(Value2)', 'Action2(Value2, Value2)']]
+        self.assertCountEqual(combinations, expected)
+
 
 if __name__ == '__main__':
     unittest.main()

chrome/test/webapps/generate_framework_tests_and_coverage.py

@@ -68,7 +68,7 @@ def generate_framework_tests_and_coverage(
         actions_csv, enums, platform_supported_actions)
 
     required_coverage_tests = read_unprocessed_coverage_tests_file(
-        coverage_required_file.readlines(), actions,
+        coverage_required_file.readlines(), actions, enums,
         action_base_name_to_default_param)
 
     required_coverage_tests = expand_parameterized_tests(

chrome/test/webapps/generate_framework_tests_and_coverage_unittest.py

@@ -69,8 +69,10 @@ def test_coverage(self):
                         as coverage_file, \
                         open(expected_coverage_filename, "r", \
                         encoding="utf-8") as expected_file:
-                    self.assertListEqual(list(coverage_file.readlines()),
-                                         list(expected_file.readlines()))
+                    self.assertListEqual(
+                        list(expected_file.readlines()),
+                        list(coverage_file.readlines()),
+                        f"file: {expected_coverage_filename}")
 
 
 if __name__ == '__main__':

chrome/test/webapps/graph_analysis_unittest.py

@@ -47,7 +47,7 @@ def test_test_generation(self):
                 actions_file.readlines(), enums, platform_supported_actions)
 
             required_coverage_tests = read_unprocessed_coverage_tests_file(
-                coverage_file.readlines(), actions,
+                coverage_file.readlines(), actions, enums,
                 action_base_name_to_default_param)
 
             required_coverage_tests = expand_parameterized_tests(

chrome/test/webapps/models.py

@@ -314,5 +314,6 @@ def generate_browsertest_filepath(self, platforms: Set[TestPlatform]):
 CoverageTestsByPlatformSet = Dict[FrozenSet[TestPlatform], List[CoverageTest]]
 CoverageTestsByPlatform = Dict[TestPlatform, List[CoverageTest]]
 ActionsByName = Dict[str, Action]
+EnumsByType = Dict[str, ArgEnum]
 PartialAndFullCoverageByBaseName = Dict[
     str, Tuple[Set[TestPlatform], Set[TestPlatform]]]

chrome/test/webapps/test_analysis_unittest.py

@@ -12,6 +12,7 @@
 from file_reading import read_platform_supported_actions
 from file_reading import read_unprocessed_coverage_tests_file
 from models import Action
+from models import EnumsByType
 from models import ActionsByName
 from models import ActionType
 from models import CoverageTest
@@ -80,6 +81,7 @@ def test_processed_coverage(self):
 
         actions: ActionsByName = {}
         action_base_name_to_default_param = {}
+        enums: EnumsByType = {}
         with open(actions_filename, "r", encoding="utf-8") as f, \
                 open(supported_actions_filename, "r", encoding="utf-8") \
                     as supported_actions_file, \
@@ -95,7 +97,8 @@ def test_processed_coverage(self):
         coverage_tests: List[CoverageTest] = []
         with open(coverage_filename, "r", encoding="utf-8") as f:
             coverage_tests = read_unprocessed_coverage_tests_file(
-                f.readlines(), actions, action_base_name_to_default_param)
+                f.readlines(), actions, enums,
+                action_base_name_to_default_param)
         coverage_tests = expand_parameterized_tests(coverage_tests)
 
         # Compare with expected
@@ -104,7 +107,8 @@ def test_processed_coverage(self):
                                           "expected_processed_coverage.md")
         with open(processed_filename, "r", encoding="utf-8") as f:
             expected_processed_tests = read_unprocessed_coverage_tests_file(
-                f.readlines(), actions, action_base_name_to_default_param)
+                f.readlines(), actions, enums,
+                action_base_name_to_default_param)
 
         # Hack for easy comparison and printing: transform coverage tests into
         # a Tuple[List[str], Set[TestPlatform]].

chrome/test/webapps/test_data/expected_coverage_cros.tsv

@@ -1,9 +1,11 @@
 # This is a generated file.
-# Full coverage: 31%, with partial coverage: 31%
+# Full coverage: 70%, with partial coverage: 70%
 state_change_a_Chicken🌕	check_a_Chicken🌕
 state_change_a_Chicken🌕	check_b_Chicken_Green🌕
 state_change_b_Chicken_Green🌑	check_a_Chicken🌑
 state_change_b_Chicken_Green🌑	check_b_Chicken_Green🌑
-state_change_a_Dog🌑	check_b_Dog_Red🌑
+state_change_a_Dog🌕	check_b_Dog_Red🌕
 state_change_a_Chicken🌕	state_change_b_Chicken_Red🌑	check_a_Chicken🌑
-state_change_a_Dog🌑	state_change_a_Chicken🌑	check_b_Chicken_Green🌑
+state_change_a_Dog🌕	state_change_a_Chicken🌕	check_b_Chicken_Green🌕
+state_change_a_Dog🌕	check_a_Chicken🌕
+state_change_a_Dog🌕	check_a_Dog🌕

chrome/test/webapps/test_data/expected_coverage_linux.tsv

@@ -5,3 +5,5 @@ state_change_a_Chicken🌕	check_b_Chicken_Green🌕
 state_change_b_Chicken_Green🌕	check_a_Chicken🌕
 state_change_b_Chicken_Green🌕	check_b_Chicken_Green🌕
 state_change_a_Dog🌕	check_b_Dog_Red🌕
+state_change_a_Dog🌕	check_a_Chicken🌕
+state_change_a_Dog🌕	check_a_Dog🌕

chrome/test/webapps/test_data/expected_coverage_mac.tsv

@@ -5,3 +5,5 @@ state_change_a_Chicken🌕	check_b_Chicken_Green🌓
 state_change_b_Chicken_Green🌕	check_a_Chicken🌓
 state_change_b_Chicken_Green🌕	check_b_Chicken_Green🌓
 state_change_a_Dog🌕	check_b_Dog_Red🌓
+state_change_a_Dog🌕	check_a_Chicken🌓
+state_change_a_Dog🌕	check_a_Dog🌓

chrome/test/webapps/test_data/expected_coverage_win.tsv

@@ -1,7 +1,9 @@
 # This is a generated file.
-# Full coverage: 60%, with partial coverage: 60%
+# Full coverage: 71%, with partial coverage: 71%
 state_change_a_Chicken🌑	check_a_Chicken🌑
 state_change_a_Chicken🌑	check_b_Chicken_Green🌑
 state_change_b_Chicken_Green🌕	check_a_Chicken🌕
 state_change_b_Chicken_Green🌕	check_b_Chicken_Green🌕
 state_change_a_Dog🌕	check_b_Dog_Red🌕
+state_change_a_Dog🌕	check_a_Chicken🌕
+state_change_a_Dog🌕	check_a_Dog🌕

chrome/test/webapps/test_data/expected_processed_coverage.md

@@ -13,3 +13,7 @@
 
 # Generated from the fourth test:
 | C | state_change_a(Dog) |  state_change_a(Chicken) |  check_b(Chicken, Green) |
+
+# Generated from fifth test
+| MWLC | state_change_a(Dog) | check_a(Dog) |
+| MWLC | state_change_a(Dog) | check_a(Chicken) |

chrome/test/webapps/test_data/expected_test_txt/tests_default_cros.txt

@@ -1,10 +1,12 @@
-IN_PROC_BROWSER_TEST_F(TestNameCros, WebAppIntegration_3Dog_2DogRed_3Chicken_2ChickenGreen) {
+IN_PROC_BROWSER_TEST_F(TestNameCros, WebAppIntegration_3Dog_2DogRed_1Chicken_1Dog_3Chicken_2ChickenGreen) {
   // Test contents are generated by script. Please do not modify!
   // See `docs/webapps/why-is-this-test-failing.md` or
   // `docs/webaps/integration-testing-framework` for more info.
   // Sheriffs: Disabling this test is supported.
   helper_.StateChangeA(Animal::kDog);
   helper_.CheckB(Animal::kDog, Color::kRed);
+  helper_.CheckA(Animal::kChicken);
+  helper_.CheckA(Animal::kDog);
   helper_.StateChangeA(Animal::kChicken);
   helper_.CheckB(Animal::kChicken, Color::kGreen);
 }

chrome/test/webapps/test_data/expected_test_txt/tests_default_mac_win_linux.txt

@@ -7,11 +7,13 @@ IN_PROC_BROWSER_TEST_F(TestNameMacWinLinux, WebAppIntegration_4ChickenGreen_1Chi
   helper_.CheckA(Animal::kChicken);
   helper_.CheckB(Animal::kChicken, Color::kGreen);
 }
-IN_PROC_BROWSER_TEST_F(TestNameMacWinLinux, WebAppIntegration_3Dog_2DogRed) {
+IN_PROC_BROWSER_TEST_F(TestNameMacWinLinux, WebAppIntegration_3Dog_2DogRed_1Chicken_1Dog) {
   // Test contents are generated by script. Please do not modify!
   // See `docs/webapps/why-is-this-test-failing.md` or
   // `docs/webaps/integration-testing-framework` for more info.
   // Sheriffs: Disabling this test is supported.
   helper_.StateChangeA(Animal::kDog);
   helper_.CheckB(Animal::kDog, Color::kRed);
+  helper_.CheckA(Animal::kChicken);
+  helper_.CheckA(Animal::kDog);
 }

chrome/test/webapps/test_data/test_unprocessed_coverage.md

@@ -8,3 +8,6 @@
 # Tests only for ChromeOS
 | C | state_change_a | state_change_b(Chicken, Red) | check_a |
 | C | state_change_a(Dog) | state_change_a(Chicken) | check_b(Chicken, Green) |
+
+# This test should generate 2 processed tests:
+| MWLC | changes(Dog) | check_a(Animal::All) |

chrome/test/webapps/test_data/tests_change_b_mac_win_linux.cc

@@ -13,11 +13,14 @@ IN_PROC_BROWSER_TEST_F(TestNameMacWinLinux,
   helper_.CheckB(Animal::kChicken, Color::kGreen);
 }
 
-IN_PROC_BROWSER_TEST_F(TestNameMacWinLinux, WebAppIntegration_3Dog_2DogRed) {
+IN_PROC_BROWSER_TEST_F(TestNameMacWinLinux,
+                       WebAppIntegration_3Dog_2DogRed_1Chicken_1Dog) {
   // Test contents are generated by script. Please do not modify!
   // See `docs/webapps/why-is-this-test-failing.md` or
   // `docs/webapps/integration-testing-framework` for more info.
   // Sheriffs: Disabling this test is supported.
   helper_.StateChangeA(Animal::kDog);
   helper_.CheckB(Animal::kDog, Color::kRed);
+  helper_.CheckA(Animal::kChicken);
+  helper_.CheckA(Animal::kDog);
 }