From 06caebe6b6db58dadadd075c8acf477bec83e0cc Mon Sep 17 00:00:00 2001 From: C Freeman Date: Fri, 26 Apr 2024 11:40:07 -0400 Subject: [PATCH] Python scripting: Add a test plan generator script (#32718) * Python scriptiong: Add a test plan generator script * Fix linter * Restyled by isort * fix spacing * update spacing on expected outcome * remove init file - it snuck in, but isn't needed * Add documnetation and better error reporting. * make print a little nicer for help * linter... * Restyled by isort * make defined steps function public --------- Co-authored-by: Restyled.io --- src/python_testing/matter_testing_support.py | 8 +- .../test_plan_table_generator.py | 95 +++++++++++++++++++ 2 files changed, 99 insertions(+), 4 deletions(-) create mode 100755 src/python_testing/test_plan_table_generator.py diff --git a/src/python_testing/matter_testing_support.py b/src/python_testing/matter_testing_support.py index 9ec12dca5fb081..4861cae7ea8c61 100644 --- a/src/python_testing/matter_testing_support.py +++ b/src/python_testing/matter_testing_support.py @@ -684,10 +684,10 @@ def get_test_steps(self, test: str) -> list[TestStep]: in order using self.step(number), where number is the test_plan_number from each TestStep. ''' - steps = self._get_defined_test_steps(test) + steps = self.get_defined_test_steps(test) return [TestStep(1, "Run entire test")] if steps is None else steps - def _get_defined_test_steps(self, test: str) -> list[TestStep]: + def get_defined_test_steps(self, test: str) -> list[TestStep]: steps_name = 'steps_' + test[5:] try: fn = getattr(self, steps_name) @@ -781,7 +781,7 @@ def setup_test(self): self.step_skipped = False if self.runner_hook and not self.is_commissioning: test_name = self.current_test_info.name - steps = self._get_defined_test_steps(test_name) + steps = self.get_defined_test_steps(test_name) num_steps = 1 if steps is None else len(steps) filename = inspect.getfile(self.__class__) desc = self.get_test_desc(test_name) @@ -977,7 +977,7 @@ def on_pass(self, record): self.runner_hook.step_success(logger=None, logs=None, duration=step_duration, request=None) # TODO: this check could easily be annoying when doing dev. flag it somehow? Ditto with the in-order check - steps = self._get_defined_test_steps(record.test_name) + steps = self.get_defined_test_steps(record.test_name) if steps is None: # if we don't have a list of steps, assume they were all run all_steps_run = True diff --git a/src/python_testing/test_plan_table_generator.py b/src/python_testing/test_plan_table_generator.py new file mode 100755 index 00000000000000..bed4fa3eaa747e --- /dev/null +++ b/src/python_testing/test_plan_table_generator.py @@ -0,0 +1,95 @@ +#!/usr/bin/env -S python3 -B +# +# Copyright (c) 2024 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import importlib +import logging +import os +import sys +from pathlib import Path + +import click +from matter_testing_support import MatterTestConfig, generate_mobly_test_config + + +def indent_multiline(multiline: str, num_spaces: int) -> str: + ''' Indents subsequent lines of a multiline string by num_spaces spaces''' + s = multiline.split('\n') + s = [(num_spaces * ' ' + line.lstrip()).rstrip() for line in s] + return '\n'.join(s).lstrip() + + +@click.command() +@click.argument('filename', type=click.Path(exists=True)) +@click.argument('classname', type=str) +@click.argument('test', type=str) +def main(filename, classname, test): + ''' + This script generates the Test Procedure table for the test plans document + from the python script steps. In order to use this generator, the test + automation script conform to the following requirements: + + - automated in python\n + - test implements the steps_ function to provide steps information to the TH\n + - TestStep list returned from the steps_ function includes both description and expectation fields\n + - test does not gate any steps on PICS (top level PICS ok)\n + + + Usage: test_plan_table_generator.py filename classname test + + filename - name of the file where the test is automated\n + classname - name of the MatterBaseTest class\n + test - name of the test to generate the table for (include the test_ portion)\n + ''' + try: + module = importlib.import_module(Path(os.path.basename(filename)).stem) + except ModuleNotFoundError: + logging.error(f'Unable to load python module from {filename}. Please ensure this is a valid python file path') + return -1 + + try: + test_class = getattr(module, classname) + except AttributeError: + logging.error(f'Unable to load the test class {classname}. Please ensure this class is implemented in {filename}') + return -1 + + config = generate_mobly_test_config(MatterTestConfig()) + test_instance = test_class(config) + steps = test_instance.get_defined_test_steps(test) + if not steps: + logging.error(f'Unable to find steps for test {test}. Please ensure the steps_ function is implemented') + return -1 + + indent = 6 + header_num = f'{"**#**":<{indent}}' + header_num_step = f'|{header_num} |*TestStep* ' + s = ('[cols="5%,45%,45%"]\n' + '|===\n' + f'{header_num_step}|*Expected Outcome*\n') + for step in steps: + step_num = f'|{step.test_plan_number:<{indent}}a|' + s += f'{step_num}{indent_multiline(step.description, len(step_num))}\n' + + padding = (len(header_num_step) - 1) * ' ' + # add 2 to indent for a| at start + s += f'{padding}a|{indent_multiline(step.expectation, len(padding)+2)}\n\n' + s += '|===\n' + + print(s) + + +if __name__ == "__main__": + sys.exit(main())