Add all passing chip-repl yaml tests to CI along with some fixes requ…

…ired (#24597)

* Add all passing chip-repl yaml tests to CI

This will prevent future breakages like some recently added due to
incorrect naming.

* Address PR comment

* Address PR comment

* Remove test that is now failing

* Remove test that are no longer passing

.github/workflows/tests.yaml

@@ -214,7 +214,7 @@ jobs:
                      --bridge-app ./out/linux-x64-bridge-${BUILD_VARIANT}/chip-bridge-app \
                   "
             - name: Run Tests using chip-repl
-              timeout-minutes: 5
+              timeout-minutes: 10
               run: |
                   ./scripts/run_in_build_env.sh \
                   "./scripts/tests/run_test_suite.py \

scripts/py_matter_yamltests/matter_yamltests/parser.py

@@ -400,6 +400,8 @@ def __init__(self, test: _TestStepWithPlaceholders, runtime_config_variable_stor
         self.response = copy.deepcopy(test.response_with_placeholders)
         self._update_placeholder_values(self.arguments)
         self._update_placeholder_values(self.response)
+        self._test.node_id = self._config_variable_substitution(
+            self._test.node_id)
         test.update_arguments(self.arguments)
         test.update_response(self.response)
 

scripts/tests/chiptest/__init__.py

@@ -77,8 +77,97 @@ def tests_with_command(chip_tool: str, is_manual: bool):
 # parser/runner reaches parity with the code gen version.
 def _hardcoded_python_yaml_tests():
     currently_supported_yaml_tests = [
-        "TestClusterComplexTypes.yaml",
+        "Test_TC_ACL_1_1.yaml",
+        "Test_TC_ACL_2_1.yaml",
+        "Test_TC_BOOL_1_1.yaml",
+        "Test_TC_ACT_1_1.yaml",
+        "Test_TC_BIND_1_1.yaml",
+        "Test_TC_OPCREDS_1_2.yaml",
+        "Test_TC_BINFO_1_1.yaml",
+        "Test_TC_DESC_1_1.yaml",
+        "Test_TC_DLOG_1_1.yaml",
+        "Test_TC_FLW_1_1.yaml",
+        "Test_TC_FLW_2_1.yaml",
+        "Test_TC_FLABEL_1_1.yaml",
+        "Test_TC_CGEN_1_1.yaml",
+        "Test_TC_DGGEN_1_1.yaml",
+        "Test_TC_I_1_1.yaml",
+        "Test_TC_I_2_1.yaml",
+        "Test_TC_ILL_1_1.yaml",
+        "Test_TC_ILL_2_1.yaml",
+        "Test_TC_LVL_2_1.yaml",
+        "Test_TC_LVL_2_2.yaml",
+        "Test_TC_LCFG_1_1.yaml",
+        "Test_TC_LTIME_1_2.yaml",
+        "Test_TC_LOWPOWER_1_1.yaml",
+        "Test_TC_WAKEONLAN_1_5.yaml",
+        "Test_TC_AUDIOOUTPUT_1_8.yaml",
+        "Test_TC_TGTNAV_1_9.yaml",
+        "Test_TC_TGTNAV_8_2.yaml",
+        "Test_TC_APBSC_1_10.yaml",
+        "Test_TC_ALOGIN_1_12.yaml",
+        "Test_TC_KEYPADINPUT_3_2.yaml",
+        "Test_TC_KEYPADINPUT_3_3.yaml",
+        "Test_TC_APPLAUNCHER_3_5.yaml",
+        "Test_TC_APPLAUNCHER_3_6.yaml",
+        "Test_TC_MEDIAINPUT_3_10.yaml",
+        "Test_TC_MEDIAINPUT_3_11.yaml",
+        "Test_TC_CHANNEL_5_1.yaml",
+        "Test_TC_CONTENTLAUNCHER_10_1.yaml",
+        "Test_TC_OCC_1_1.yaml",
+        "Test_TC_PSCFG_1_1.yaml",
+        "Test_TC_PSCFG_2_1.yaml",
+        "Test_TC_RH_1_1.yaml",
+        "Test_TC_RH_2_1.yaml",
+        "Test_TC_SWTCH_2_1.yaml",
+        "Test_TC_TMP_1_1.yaml",
+        "Test_TC_TMP_2_1.yaml",
+        "Test_TC_TSUIC_1_1.yaml",
+        "Test_TC_TSUIC_2_1.yaml",
+        "Test_TC_DGTHREAD_2_2.yaml",
+        "Test_TC_DGTHREAD_2_4.yaml",
+        "Test_TC_ULABEL_1_1.yaml",
+        "Test_TC_ULABEL_2_1.yaml",
+        "Test_TC_ULABEL_2_2.yaml",
+        "Test_TC_ULABEL_2_3.yaml",
+        "Test_TC_ULABEL_2_4.yaml",
+        "Test_TC_DGWIFI_2_3.yaml",
+        "TV_TargetNavigatorCluster.yaml",
+        "TV_AudioOutputCluster.yaml",
+        "TV_ApplicationLauncherCluster.yaml",
+        "TV_KeypadInputCluster.yaml",
+        "TV_AccountLoginCluster.yaml",
+        "TV_WakeOnLanCluster.yaml",
+        "TV_ApplicationBasicCluster.yaml",
+        "TV_ChannelCluster.yaml",
+        "TV_LowPowerCluster.yaml",
+        "TV_ContentLauncherCluster.yaml",
+        "TV_MediaInputCluster.yaml",
+        "TestCluster.yaml",
         "TestConstraints.yaml",
+        "TestSaveAs.yaml",
+        "TestConfigVariables.yaml",
+        "TestFabricRemovalWhileSubscribed.yaml",
+        "TestIdentifyCluster.yaml",
+        "TestSelfFabricRemoval.yaml",
+        "TestBinding.yaml",
+        "TestUserLabelClusterConstraints.yaml",
+        "TestFanControl.yaml",
+        "TestAccessControlConstraints.yaml",
+        "TestCommissioningWindow.yaml",
+        "TestSubscribe_OnOff.yaml",
+        "TestClusterComplexTypes.yaml",
+        "TestGroupsCluster.yaml",
+        "TestOperationalCredentialsCluster.yaml",
+        "Test_TC_AUDIOOUTPUT_7_1.yaml",
+        "Test_TC_BOOL_2_1.yaml",
+        "Test_TC_OO_2_1.yaml",
+        "Test_TC_TGTNAV_8_1.yaml",
+        "Test_TC_WNCV_2_3.yaml",
+        "Test_TC_WNCV_4_3.yaml",
+        "Test_TC_WNCV_4_4.yaml",
+        "DL_Schedules.yaml",
+        "DL_UsersAndCredentials.yaml",
     ]
 
     for name in currently_supported_yaml_tests:

scripts/tests/chiptest/yamltest_with_chip_repl_tester.py

@@ -18,6 +18,7 @@
 import glob
 import os
 import tempfile
+import traceback
 
 # isort: off
 
@@ -88,28 +89,32 @@ def main(setup_code, yaml_path, node_id):
         dev_ctrl = ca_list[0].adminList[0].NewController()
         dev_ctrl.CommissionWithCode(setup_code, node_id)
 
-        # Creating Cluster definition.
-        cluster_xml_filenames = glob.glob(_CLUSTER_XML_DIRECTORY_PATH + '/*/*.xml', recursive=False)
-        cluster_xml_filenames.sort(key=functools.cmp_to_key(_sort_with_global_attribute_first))
-        sources = [ParseSource(source=name) for name in cluster_xml_filenames]
-        clusters_definitions = SpecDefinitions(sources)
-
-        # Parsing YAML test and setting up chip-repl yamltests runner.
-        yaml = TestParser(yaml_path, None, clusters_definitions)
-        runner = ReplTestRunner(clusters_definitions, certificate_authority_manager, dev_ctrl)
-
-        # Executing and validating test
-        for test_step in yaml.tests:
-            test_action = runner.encode(test_step)
-            # TODO if test_action is None we should see if it is a pseudo cluster.
-            if test_action is not None:
-                response = runner.execute(test_action)
-                decoded_response = runner.decode(response)
-                post_processing_result = test_step.post_process_response(decoded_response)
-                if not post_processing_result.is_success():
-                    exit(-2)
-            else:
-                exit(-2)
+        try:
+            # Creating Cluster definition.
+            cluster_xml_filenames = glob.glob(_CLUSTER_XML_DIRECTORY_PATH + '/*/*.xml', recursive=False)
+            cluster_xml_filenames.sort(key=functools.cmp_to_key(_sort_with_global_attribute_first))
+            sources = [ParseSource(source=name) for name in cluster_xml_filenames]
+            clusters_definitions = SpecDefinitions(sources)
+
+            # Parsing YAML test and setting up chip-repl yamltests runner.
+            yaml = TestParser(yaml_path, None, clusters_definitions)
+            runner = ReplTestRunner(clusters_definitions, certificate_authority_manager, dev_ctrl)
+
+            # Executing and validating test
+            for test_step in yaml.tests:
+                test_action = runner.encode(test_step)
+                # TODO if test_action is None we should see if it is a pseudo cluster.
+                if test_action is not None:
+                    response = runner.execute(test_action)
+                    decoded_response = runner.decode(response)
+                    post_processing_result = test_step.post_process_response(decoded_response)
+                    if not post_processing_result.is_success():
+                        raise Exception(f'Test step failed {test_step.label}')
+                else:
+                    raise Exception(f'Failed to encode test step {test_step.label}')
+        except Exception:
+            print(traceback.format_exc())
+            exit(-2)
 
         runner.shutdown()
         # Tearing down chip stack. If not done in the correct order test will fail.

scripts/tests/run_test_suite.py

@@ -262,6 +262,7 @@ def cmd_run(context, iterations, all_clusters_app, lock_app, ota_provider_app, o
                 if context.obj.dry_run:
                     logging.info("Would run test %s:" % test.name)
 
+                logging.info('%-20s - Starting test' % (test.name))
                 test.Run(runner, apps_register, paths, pics_file, test_timeout_seconds, context.obj.dry_run)
                 test_end = time.monotonic()
                 logging.info('%-20s - Completed in %0.2f seconds' %

src/controller/python/chip/yaml/data_model_lookup.py

@@ -20,6 +20,13 @@
 import chip.clusters as Clusters
 
 
+def _case_insensitive_getattr(object, attr_name, default):
+    for attr in dir(object):
+        if attr.lower() == attr_name.lower():
+            return getattr(object, attr)
+    return default
+
+
 class DataModelLookup(ABC):
     @abstractmethod
     def get_cluster(self, cluster: str):
@@ -41,27 +48,27 @@ def get_event(self, cluster: str, event: str):
 class PreDefinedDataModelLookup(DataModelLookup):
     def get_cluster(self, cluster: str):
         try:
-            return getattr(Clusters, cluster, None)
+            return _case_insensitive_getattr(Clusters, cluster, None)
         except AttributeError:
             return None
 
     def get_command(self, cluster: str, command: str):
         try:
-            commands = getattr(Clusters, cluster, None).Commands
-            return getattr(commands, command, None)
+            commands = _case_insensitive_getattr(Clusters, cluster, None).Commands
+            return _case_insensitive_getattr(commands, command, None)
         except AttributeError:
             return None
 
     def get_attribute(self, cluster: str, attribute: str):
         try:
-            attributes = getattr(Clusters, cluster, None).Attributes
-            return getattr(attributes, attribute, None)
+            attributes = _case_insensitive_getattr(Clusters, cluster, None).Attributes
+            return _case_insensitive_getattr(attributes, attribute, None)
         except AttributeError:
             return None
 
     def get_event(self, cluster: str, event: str):
         try:
-            events = getattr(Clusters, cluster, None).Events
-            return getattr(events, event, None)
+            events = _case_insensitive_getattr(Clusters, cluster, None).Events
+            return _case_insensitive_getattr(events, event, None)
         except AttributeError:
             return None

src/controller/python/chip/yaml/runner.py

@@ -644,8 +644,13 @@ def decode(self, result: _ActionResult):
             if attribute is None:
                 # When we cannot find the attribute it is because it is a global attribute like
                 # FeatureMap. Fortunately for these types we can get away with using
-                # 'response.value' directly for the time being.
-                decoded_response['value'] = response.value
+                # 'response.value' directly if it is a list and mapping to int if not a list.
+                if isinstance(response.value, list):
+                    decoded_response['value'] = response.value
+                else:
+                    decoded_response['value'] = Converter.from_data_model_to_test_definition(
+                        self._test_spec_definition, cluster_name, int, response.value)
+
             else:
                 decoded_response['value'] = Converter.from_data_model_to_test_definition(
                     self._test_spec_definition, cluster_name, attribute.definition, response.value)