Skip to content

Commit

Permalink
[MatterYamlTests][darwin-framework-tool] Use a list of tests to run f… (
Browse files Browse the repository at this point in the history 
#31792)
  • Loading branch information
@vivien-apple @pull
vivien-apple authored and pull[bot] committed Feb 22, 2024
1 parent 3b8dfb9 commit 1048129
Show file tree
Hide file tree
Showing 5 changed files with 109 additions and 49 deletions.
1 change: 1 addition & 0 deletions .github/workflows/darwin-tests.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ jobs:
run: |
./scripts/run_in_build_env.sh \
"./scripts/tests/run_test_suite.py \
--runner darwin_framework_tool_python \
--chip-tool ./out/darwin-x64-darwin-framework-tool-${BUILD_VARIANT_FRAMEWORK_TOOL}/darwin-framework-tool \
--target-skip-glob '{TestAccessControlConstraints}' \
run \
Expand Down
2 changes: 1 addition & 1 deletion examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/decoder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,7 @@ def run(self, specs, value, cluster_name: str, typename: str, array: bool):
provided_field_name = provided_field_name[0].lower(
) + provided_field_name[1:]

if provided_field_name in value:
if provided_field_name in value and provided_field_name != field_name:
value[field_name] = self.run(
specs,
value[provided_field_name],
Expand Down
69 changes: 66 additions & 3 deletions scripts/tests/chiptest/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,60 @@ def _GetInDevelopmentTests() -> Set[str]:
}


def _GetDarwinFrameworkToolUnsupportedTests() -> Set[str]:
"""Tests that fail in darwin-framework-tool for some reason"""
return {
"DL_LockUnlock", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_AddNewFabricFromExistingFabric", # darwin-framework-tool does not support the GetCommissionerRootCertificate command.
# The name of the arguments once converted differs for chip-tool and darwin-framework-tool (attribute-ids vs attribute-id. See #31934)
"TestAttributesById",
"TestBasicInformation", # darwin-framework-tool does not support writing readonly attributes by name
"TestClusterComplexTypes", # Darwin framework has no way to represent a present but null optional nullable field.
# When reading TestFabricScoped in TestClusterMultiFabric, the result differs because of missing fields that have been declared in the YAML step with null value to workaround some limitation of the test harness (#29110)
"TestClusterMultiFabric",
"TestCommandsById", # darwin-framework-tool does not support writing readonly attributes by name
"TestDiagnosticLogs", # darwin-framework-tool does not implement a BDXTransferServerDelegate
"TestDiscovery", # darwin-framework-tool does not support dns-sd commands.
"TestEvents", # darwin-framework-tool does not currently support reading or subscribing to Events
"TestEventsById", # darwin-framework-tool does not currently support reading or subscribing to Events
"TestGroupMessaging", # darwin-framework-tool does not support group commands.
"TestIcdManagementCluster", # darwin-framework-tool does not support ICD registration
"TestUnitTestingClusterMei", # darwin-framework-tool does not currently support reading or subscribing to Events
"TestReadNoneSubscribeNone", # darwin-framework-tool does not supports those commands.

"Test_TC_ACE_1_6", # darwin-framework-tool does not support group commands.
"Test_TC_ACL_2_5", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_ACL_2_6", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_ACL_2_7", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_ACL_2_8", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_ACL_2_9", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_ACL_2_10", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_BINFO_2_1", # darwin-framework-tool does not support writing readonly attributes by name
"Test_TC_BINFO_2_2", # darwin-framework-tool does not currently support reading or subscribing to Events
# The name of the arguments once converted differs for chip-tool and darwin-framework-tool (attribute-ids vs attribute-id. See #31934)
"Test_TC_BRBINFO_2_1",
"Test_TC_DGGEN_2_3", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_DRLK_2_1", # darwin-framework-tool does not support writing readonly attributes by name
"Test_TC_DGTHREAD_2_1", # Thread Network Diagnostics is not implemented under darwin.
"Test_TC_DGTHREAD_2_2", # Thread Network Diagnostics is not implemented under darwin.
"Test_TC_DGTHREAD_2_3", # Thread Network Diagnostics is not implemented under darwin.
"Test_TC_DGTHREAD_2_4", # Thread Network Diagnostics is not implemented under darwin.
"Test_TC_FLABEL_2_1", # darwin-framework-tool does not support writing readonly attributes by name
"Test_TC_GRPKEY_2_1", # darwin-framework-tool does not support writing readonly attributes by name
"Test_TC_LCFG_2_1", # darwin-framework-tool does not support writing readonly attributes by name
"Test_TC_OPCREDS_3_7", # darwin-framework-tool does not support the GetCommissionerRootCertificate command.
"Test_TC_OPSTATE_2_4", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_SMOKECO_2_2", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_SMOKECO_2_3", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_SMOKECO_2_4", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_SMOKECO_2_5", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_SMOKECO_2_6", # darwin-framework-tool does not currently support reading or subscribing to Events
"Test_TC_SC_4_1", # darwin-framework-tool does not support dns-sd commands.
"Test_TC_SC_5_2", # darwin-framework-tool does not support group commands.
"Test_TC_S_2_3", # darwin-framework-tool does not support group commands.
}


def _GetChipReplUnsupportedTests() -> Set[str]:
"""Tests that fail in chip-repl for some reason"""
return {
Expand Down Expand Up @@ -284,7 +338,7 @@ def tests_with_command(chip_tool: str, is_manual: bool):
)


def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short_run_name: bool):
def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, treat_dft_unsupported_as_in_development: bool, use_short_run_name: bool):
"""
use_short_run_name should be true if we want the run_name to be "Test_ABC" instead of "some/path/Test_ABC.yaml"
"""
Expand All @@ -294,6 +348,7 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short
extra_slow_tests = _GetExtraSlowTests()
in_development_tests = _GetInDevelopmentTests()
chip_repl_unsupported_tests = _GetChipReplUnsupportedTests()
treat_dft_unsupported_as_in_development_tests = _GetDarwinFrameworkToolUnsupportedTests()
purposeful_failure_tests = _GetPurposefulFailureTests()

for path in _AllYamlTests():
Expand Down Expand Up @@ -327,6 +382,9 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short
else:
run_name = str(path)

if treat_dft_unsupported_as_in_development and run_name in treat_dft_unsupported_as_in_development_tests:
tags.add(TestTag.IN_DEVELOPMENT)

yield TestDefinition(
run_name=run_name,
name=path.stem, # `path.stem` converts "some/path/Test_ABC_1.2.yaml" to "Test_ABC.1.2"
Expand All @@ -336,12 +394,17 @@ def _AllFoundYamlTests(treat_repl_unsupported_as_in_development: bool, use_short


def AllReplYamlTests():
for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, use_short_run_name=False):
for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=True, treat_dft_unsupported_as_in_development=False, use_short_run_name=False):
yield test


def AllChipToolYamlTests():
for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, use_short_run_name=True):
for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, treat_dft_unsupported_as_in_development=False, use_short_run_name=True):
yield test


def AllDarwinFrameworkToolYamlTests():
for test in _AllFoundYamlTests(treat_repl_unsupported_as_in_development=False, treat_dft_unsupported_as_in_development=True, use_short_run_name=True):
yield test


Expand Down
58 changes: 27 additions & 31 deletions scripts/tests/chiptest/test_definition.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,10 +251,9 @@ def to_s(self):


class TestRunTime(Enum):
CHIP_TOOL_BUILTIN = auto() # run via chip-tool built-in test commands
CHIP_TOOL_PYTHON = auto() # use the python yaml test parser with chip-tool
DARWIN_FRAMEWORK_TOOL_PYTHON = auto() # use the python yaml test parser with chip-tool
CHIP_REPL_PYTHON = auto() # use the python yaml test runner
DARWIN_FRAMEWORK_TOOL_BUILTIN = auto() # run via darwin-framework-tool built-in test commands


@dataclass
Expand All @@ -281,7 +280,7 @@ def tags_str(self) -> str:
return ", ".join([t.to_s() for t in self.tags])

def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str,
timeout_seconds: typing.Optional[int], dry_run=False, test_runtime: TestRunTime = TestRunTime.CHIP_TOOL_BUILTIN):
timeout_seconds: typing.Optional[int], dry_run=False, test_runtime: TestRunTime = TestRunTime.CHIP_TOOL_PYTHON):
"""
Executes the given test case using the provided runner for execution.
"""
Expand Down Expand Up @@ -333,8 +332,6 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str,
# so it will be commissionable again.
app.factoryReset()

tool_cmd = paths.chip_tool if test_runtime != TestRunTime.CHIP_TOOL_PYTHON else paths.chip_tool_with_python_cmd

if dry_run:
tool_storage_dir = None
tool_storage_args = []
Expand All @@ -350,38 +347,37 @@ def Run(self, runner, apps_register, paths: ApplicationPaths, pics_file: str,
app.start()
setupCode = app.setupCode

pairing_cmd = tool_cmd + ['pairing', 'code', TEST_NODE_ID, setupCode]
test_cmd = tool_cmd + ['tests', self.run_name] + ['--PICS', pics_file]
if test_runtime == TestRunTime.CHIP_TOOL_PYTHON:
if test_runtime == TestRunTime.CHIP_REPL_PYTHON:
chip_repl_yaml_tester_cmd = paths.chip_repl_yaml_tester_cmd
python_cmd = chip_repl_yaml_tester_cmd + \
['--setup-code', setupCode] + ['--yaml-path', self.run_name] + ["--pics-file", pics_file]
if dry_run:
logging.info(" ".join(python_cmd))
else:
runner.RunSubprocess(python_cmd, name='CHIP_REPL_YAML_TESTER',
dependencies=[apps_register], timeout_seconds=timeout_seconds)
else:
pairing_cmd = paths.chip_tool_with_python_cmd + ['pairing', 'code', TEST_NODE_ID, setupCode]
test_cmd = paths.chip_tool_with_python_cmd + ['tests', self.run_name] + ['--PICS', pics_file]
server_args = ['--server_path', paths.chip_tool[-1]] + \
['--server_arguments', 'interactive server' +
(' ' if len(tool_storage_args) else '') + ' '.join(tool_storage_args)]
pairing_cmd += server_args
test_cmd += server_args
elif test_runtime == TestRunTime.CHIP_TOOL_BUILTIN:
pairing_cmd += tool_storage_args
test_cmd += tool_storage_args

if dry_run:
# Some of our command arguments have spaces in them, so if we are
# trying to log commands people can run we should quote those.
def quoter(arg): return f"'{arg}'" if ' ' in arg else arg
logging.info(" ".join(map(quoter, pairing_cmd)))
logging.info(" ".join(map(quoter, test_cmd)))
elif test_runtime == TestRunTime.CHIP_REPL_PYTHON:
chip_repl_yaml_tester_cmd = paths.chip_repl_yaml_tester_cmd
python_cmd = chip_repl_yaml_tester_cmd + \
['--setup-code', app.setupCode] + ['--yaml-path', self.run_name] + ["--pics-file", pics_file]
runner.RunSubprocess(python_cmd, name='CHIP_REPL_YAML_TESTER',
dependencies=[apps_register], timeout_seconds=timeout_seconds)
else:
runner.RunSubprocess(pairing_cmd,
name='PAIR', dependencies=[apps_register])

runner.RunSubprocess(
test_cmd,
name='TEST', dependencies=[apps_register],
timeout_seconds=timeout_seconds)
if dry_run:
# Some of our command arguments have spaces in them, so if we are
# trying to log commands people can run we should quote those.
def quoter(arg): return f"'{arg}'" if ' ' in arg else arg
logging.info(" ".join(map(quoter, pairing_cmd)))
logging.info(" ".join(map(quoter, test_cmd)))
else:
runner.RunSubprocess(pairing_cmd,
name='PAIR', dependencies=[apps_register])
runner.RunSubprocess(
test_cmd,
name='TEST', dependencies=[apps_register],
timeout_seconds=timeout_seconds)

except Exception:
logging.error("!!!!!!!!!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!!!!!!!")
Expand Down
28 changes: 14 additions & 14 deletions scripts/tests/run_test_suite.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,8 @@ class RunContext:
)
@click.option(
'--runner',
type=click.Choice(['codegen', 'chip_repl_python', 'chip_tool_python'], case_sensitive=False),
default='codegen',
type=click.Choice(['chip_repl_python', 'chip_tool_python', 'darwin_framework_tool_python'], case_sensitive=False),
default='chip_tool_python',
help='Run YAML tests using the specified runner.')
@click.option(
'--chip-tool',
Expand All @@ -138,18 +138,18 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob,
log_fmt = '%(levelname)-7s %(message)s'
coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt=log_fmt)

runtime = TestRunTime.CHIP_TOOL_BUILTIN
runtime = TestRunTime.CHIP_TOOL_PYTHON
if runner == 'chip_repl_python':
runtime = TestRunTime.CHIP_REPL_PYTHON
elif runner == 'chip_tool_python':
runtime = TestRunTime.CHIP_TOOL_PYTHON
elif chip_tool is not None and os.path.basename(chip_tool) == "darwin-framework-tool":
runtime = TestRunTime.DARWIN_FRAMEWORK_TOOL_BUILTIN
elif runner == 'darwin_framework_tool_python':
runtime = TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON

if chip_tool is None and not runtime == TestRunTime.CHIP_REPL_PYTHON:
# non yaml tests REQUIRE chip-tool. Yaml tests should not require chip-tool
paths_finder = PathsFinder()
chip_tool = paths_finder.get('chip-tool')
if runtime == TestRunTime.CHIP_TOOL_PYTHON:
chip_tool = paths_finder.get('chip-tool')
else:
chip_tool = paths_finder.get('darwin-framework-tool')

if include_tags:
include_tags = set([TestTag.__members__[t] for t in include_tags])
Expand All @@ -160,10 +160,10 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob,
# Figures out selected test that match the given name(s)
if runtime == TestRunTime.CHIP_REPL_PYTHON:
all_tests = [test for test in chiptest.AllReplYamlTests()]
elif runtime == TestRunTime.CHIP_TOOL_PYTHON and os.path.basename(chip_tool) != "darwin-framework-tool":
all_tests = [test for test in chiptest.AllChipToolYamlTests()]
elif runtime == TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON:
all_tests = [test for test in chiptest.AllDarwinFrameworkToolYamlTests()]
else:
all_tests = [test for test in chiptest.AllChipToolTests(chip_tool)]
all_tests = [test for test in chiptest.AllChipToolYamlTests()]

tests = all_tests

Expand All @@ -178,7 +178,7 @@ def main(context, dry_run, log_level, target, target_glob, target_skip_glob,
TestTag.PURPOSEFUL_FAILURE,
}

if runtime != TestRunTime.CHIP_TOOL_PYTHON:
if runtime == TestRunTime.CHIP_REPL_PYTHON:
exclude_tags.add(TestTag.CHIP_TOOL_PYTHON_ONLY)

if 'all' not in target:
Expand Down Expand Up @@ -319,7 +319,7 @@ def cmd_run(context, iterations, all_clusters_app, lock_app, ota_provider_app, o
chip_repl_yaml_tester = paths_finder.get('yamltest_with_chip_repl_tester.py')

if chip_tool_with_python is None:
if context.obj.chip_tool and os.path.basename(context.obj.chip_tool) == "darwin-framework-tool":
if context.obj.runtime == TestRunTime.DARWIN_FRAMEWORK_TOOL_PYTHON:
chip_tool_with_python = paths_finder.get('darwinframeworktool.py')
else:
chip_tool_with_python = paths_finder.get('chiptool.py')
Expand Down

0 comments on commit 1048129

Please sign in to comment.