From 66de82a4f6b9b0b035af5e479da5888c251d7107 Mon Sep 17 00:00:00 2001 From: bruno Date: Sun, 8 Oct 2023 00:55:33 -0300 Subject: [PATCH 1/3] 78 - Parameter to exclude utters and actions --- CHANGELOG.md | 5 ++-- .../controllers/e2e_coverage_controller.py | 3 ++ .../controllers/markdown_controller.py | 1 + rasa_model_report/helpers/constants.py | 1 + rasa_model_report/main.py | 10 ++++++- tests/conftest.py | 25 +++++++++++----- tests/mocks/rasa.v2/domain.yml | 6 ++++ tests/mocks/rasa.v3/domain.yml | 6 ++++ tests/test_e2e_coverage_controller.py | 3 +- tests/test_main.py | 30 +++++++++++++++---- tests/utils.py | 10 +++++++ 11 files changed, 84 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aba1a2d..ec72c1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.3.5] - In progress... +## [1.4.0] - In progress... ### Added - [#73](https://github.com/brunohjs/rasa-model-report/issues/73) Created section with element count on model report. Additionally, now all tables elements are numbered. +- [#78](https://github.com/brunohjs/rasa-model-report/issues/78) Created `--exclude` CLI command parameter. This command is used to exclude utters and actions from end-to-end test coverage. ### Fixed - [#75](https://github.com/brunohjs/rasa-model-report/issues/75) Fixed bug with utters in actions that weren't being covered by the end-to-end tests. @@ -88,7 +89,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - [#16](https://github.com/brunohjs/rasa-model-report/issues/16) Created a handler for retrieval intents in the report. -[1.3.5]: https://github.com/brunohjs/rasa-model-report/compare/1.3.4...1.3.5 +[1.3.5]: https://github.com/brunohjs/rasa-model-report/compare/1.3.4...1.4.0 [1.3.4]: https://github.com/brunohjs/rasa-model-report/compare/1.3.3...1.3.4 [1.3.3]: https://github.com/brunohjs/rasa-model-report/compare/1.3.2...1.3.3 [1.3.2]: https://github.com/brunohjs/rasa-model-report/compare/1.3.0...1.3.2 diff --git a/rasa_model_report/controllers/e2e_coverage_controller.py b/rasa_model_report/controllers/e2e_coverage_controller.py index 253eda7..65628cc 100644 --- a/rasa_model_report/controllers/e2e_coverage_controller.py +++ b/rasa_model_report/controllers/e2e_coverage_controller.py @@ -17,6 +17,7 @@ def __init__( rasa_path: str, output_path: str, actions_path: str, + exclude: List[str], project_name: str, project_version: str, **kwargs: Dict[str, Any] @@ -38,6 +39,7 @@ def __init__( self._entities: List[str] = [] self._actions: List[str] = [] self._total_rate: float = 0 + self._exclude = exclude self.json: JsonController = JsonController(rasa_path, output_path, project_name, project_version) self._load_domain_elements() @@ -72,6 +74,7 @@ def _load_domain_elements(self) -> None: setattr(self, f"_{element}", getattr(self, f"_{element}") + data) self._actions = list(dict.fromkeys(self._actions)) self._actions = self._exclude_special_actions() + self._actions = utils.list_diff(self._actions, self._exclude) self._actions = utils.list_diff(self._actions, self.get_utters_in_actions()) def _generate(self) -> None: diff --git a/rasa_model_report/controllers/markdown_controller.py b/rasa_model_report/controllers/markdown_controller.py index cc3b80f..4af39af 100644 --- a/rasa_model_report/controllers/markdown_controller.py +++ b/rasa_model_report/controllers/markdown_controller.py @@ -61,6 +61,7 @@ def __init__( rasa_path, output_path, kwargs.get("actions_path"), + kwargs.get("exclude", []), project_name, project_version ) diff --git a/rasa_model_report/helpers/constants.py b/rasa_model_report/helpers/constants.py index 0b1cc60..cdbec64 100644 --- a/rasa_model_report/helpers/constants.py +++ b/rasa_model_report/helpers/constants.py @@ -7,3 +7,4 @@ RASA_API_URL = "http://localhost:5005" RASA_PATH = "./" RASA_VERSION = None +EXCLUDE = [] diff --git a/rasa_model_report/main.py b/rasa_model_report/main.py index 45312bb..b2d449c 100644 --- a/rasa_model_report/main.py +++ b/rasa_model_report/main.py @@ -22,6 +22,12 @@ help="Disable processing NLU sentences. NLU section will not be generated " "in the report. Required Rasa API." ) +@click.option( + "--exclude", + required=False, + help="List of utter and actions that will be exclude in the E2E test coverage. Use commas to separate items. " + "Example: utter_greet,utter_goodbye,action_listen" +) @click.help_option( "--help", "-h", @@ -101,6 +107,7 @@ def main( actions_path, disable_nlu, + exclude, model_link, no_images, output_path, path, @@ -124,6 +131,7 @@ def main( model_link=model_link, actions_path=actions_path, no_images=no_images, - precision=precision + precision=precision, + exclude=exclude.split(",") if exclude else constants.EXCLUDE ) return report diff --git a/tests/conftest.py b/tests/conftest.py index 43a7a55..75c2cb8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,8 +14,8 @@ def pytest_generate_tests(metafunc): rasa_dirs = glob.glob("tests/mocks/rasa.v*") - metafunc.fixturenames.append('rasa_path') - metafunc.parametrize('rasa_path', rasa_dirs) + metafunc.fixturenames.append("rasa_path") + metafunc.parametrize("rasa_path", rasa_dirs) @pytest.fixture(autouse=True) @@ -33,13 +33,24 @@ def load_controllers(rasa_path): project_name = "test-project" project_version = "0.0.0" rasa_version = "0.0.0" + exclude = [] utils.load_mock_payloads() controller = Controller(rasa_path, output_path, project_name, project_version) - json_controller = JsonController(rasa_path, output_path, project_name, project_version) - csv_controller = CsvController(rasa_path, output_path, project_name, project_version) - nlu_controller = NluController(rasa_path, output_path, project_name, project_version) - e2e_coverage_controller = E2ECoverageController(rasa_path, output_path, actions_path, project_name, project_version) - markdown_controller = MarkdownController(rasa_path, output_path, project_name, rasa_version, project_version) + json_controller = JsonController( + rasa_path, output_path, project_name, project_version + ) + csv_controller = CsvController( + rasa_path, output_path, project_name, project_version + ) + nlu_controller = NluController( + rasa_path, output_path, project_name, project_version + ) + e2e_coverage_controller = E2ECoverageController( + rasa_path, output_path, actions_path, exclude, project_name, project_version + ) + markdown_controller = MarkdownController( + rasa_path, output_path, project_name, rasa_version, project_version + ) pytest.controller = controller pytest.json_controller = json_controller pytest.csv_controller = csv_controller diff --git a/tests/mocks/rasa.v2/domain.yml b/tests/mocks/rasa.v2/domain.yml index 92dccec..cc94b68 100644 --- a/tests/mocks/rasa.v2/domain.yml +++ b/tests/mocks/rasa.v2/domain.yml @@ -42,6 +42,12 @@ responses: utter_test_2: - text: "Test 2" + utter_uncovered: + - text: "Uncovered utter" + + utter_another_uncovered: + - text: "Another uncovered utter" + session_config: session_expiration_time: 60 carry_over_slots_to_new_session: true diff --git a/tests/mocks/rasa.v3/domain.yml b/tests/mocks/rasa.v3/domain.yml index fa30c1f..0ad0827 100644 --- a/tests/mocks/rasa.v3/domain.yml +++ b/tests/mocks/rasa.v3/domain.yml @@ -40,6 +40,12 @@ responses: utter_test_2: - text: "Test 2" + utter_uncovered: + - text: "Uncovered utter" + + utter_another_uncovered: + - text: "Another uncovered utter" + session_config: session_expiration_time: 60 carry_over_slots_to_new_session: true diff --git a/tests/test_e2e_coverage_controller.py b/tests/test_e2e_coverage_controller.py index 196ba3b..0c22fb9 100644 --- a/tests/test_e2e_coverage_controller.py +++ b/tests/test_e2e_coverage_controller.py @@ -12,8 +12,9 @@ def execute_before_each_test(rasa_path): actions_path = None project_name = "test-project" project_version = "0.0.0" + exclude = [] e2e_coverage_controller = E2ECoverageController( - rasa_path, output_path, actions_path, project_name, project_version + rasa_path, output_path, actions_path, exclude, project_name, project_version ) pytest.e2e_coverage_controller = e2e_coverage_controller yield diff --git a/tests/test_main.py b/tests/test_main.py index dc79c4f..dfb8977 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -7,13 +7,33 @@ from tests import utils +@responses.activate +def test_main_with_exclude(rasa_path): + runner = CliRunner() + result = runner.invoke( + main, + ["--path", rasa_path, "--exclude", "utter_uncovered,utter_another_uncovered"], + ) + assert os.path.isfile("model_report.md") + assert utils.check_model_report_sections("model_report.md") is True + assert utils.check_model_report_images("model_report.md") is True + assert ( + utils.check_model_report_text( + "model_report.md", ["utter_uncovered", "utter_another_uncovered"] + ) + is False + ) + assert result.exit_code == 0 + assert result.output == "" + + @responses.activate def test_main_with_valid_path(rasa_path): runner = CliRunner() result = runner.invoke(main, ["--path", rasa_path]) assert os.path.isfile("model_report.md") - assert utils.check_model_report_sections("model_report.md") - assert utils.check_model_report_images("model_report.md") + assert utils.check_model_report_sections("model_report.md") is True + assert utils.check_model_report_images("model_report.md") is True assert result.exit_code == 0 assert result.output == "" @@ -23,7 +43,7 @@ def test_main_with_invalid_path(): utils.load_mock_payloads() runner = CliRunner() result = runner.invoke(main, []) - assert not os.path.isfile("model_report.md") + assert os.path.isfile("model_report.md") is False assert result.exit_code == 0 assert result.output == "" @@ -33,8 +53,8 @@ def test_main_with_no_images(rasa_path): utils.load_mock_payloads() runner = CliRunner() result = runner.invoke(main, ["--path", rasa_path, "--no-images"]) - assert os.path.isfile("model_report.md") - assert not utils.check_model_report_images("model_report.md") + assert os.path.isfile("model_report.md") is True + assert utils.check_model_report_images("model_report.md") is False assert result.exit_code == 0 assert result.output == "" diff --git a/tests/utils.py b/tests/utils.py index a51b9a7..db3ed07 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -77,3 +77,13 @@ def check_model_report_images(model_report_path): "/DIETClassifier_confusion_matrix.png" in file_data and \ "/story_confusion_matrix.png" in file_data and \ "/intent_histogram.png" in file_data + + +def check_model_report_text(model_report_path, texts): + file = open(model_report_path, encoding="utf-8") + file_data = file.read() + file.close() + if isinstance(texts, str): + return texts in file_data + else: + return all([text in file_data for text in texts]) From 850c74ac6bf8eda1d84db619e37d20d3e22ca431 Mon Sep 17 00:00:00 2001 From: bruno Date: Sun, 8 Oct 2023 01:17:07 -0300 Subject: [PATCH 2/3] 78 - Add -e option --- README.md | 3 +++ rasa_model_report/main.py | 5 ++++- tests/test_main.py | 4 ++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 470e5bd..4fb066c 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,9 @@ There are parameters that can be used. Available options are below: path) --disable-nlu Disable processing NLU sentences. NLU section will not be generated in the report. Required Rasa API. +-e, --exclude LIST List of utter and actions that will be exclude in + the E2E test coverage. Use commas to separate items. + Example: utter_greet,utter_goodbye,action_listen -h, --help Show this help message. --model-link TEXT Model download link. It's only displayed in the report to model download. diff --git a/rasa_model_report/main.py b/rasa_model_report/main.py index b2d449c..47ae906 100644 --- a/rasa_model_report/main.py +++ b/rasa_model_report/main.py @@ -24,7 +24,10 @@ ) @click.option( "--exclude", + "-e", required=False, + multiple=True, + type=list, help="List of utter and actions that will be exclude in the E2E test coverage. Use commas to separate items. " "Example: utter_greet,utter_goodbye,action_listen" ) @@ -132,6 +135,6 @@ def main( actions_path=actions_path, no_images=no_images, precision=precision, - exclude=exclude.split(",") if exclude else constants.EXCLUDE + exclude=[item for row in exclude for item in row.split(",")] ) return report diff --git a/tests/test_main.py b/tests/test_main.py index dfb8977..4cb1075 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -12,9 +12,9 @@ def test_main_with_exclude(rasa_path): runner = CliRunner() result = runner.invoke( main, - ["--path", rasa_path, "--exclude", "utter_uncovered,utter_another_uncovered"], + ["--path", rasa_path, "--exclude", "utter_uncovered,utter_another_uncovered", "-e", "utter_goodbay"], ) - assert os.path.isfile("model_report.md") + assert os.path.isfile("model_report.md") is True assert utils.check_model_report_sections("model_report.md") is True assert utils.check_model_report_images("model_report.md") is True assert ( From 7d8eb919555627025a90aa731f75c0d1618be24a Mon Sep 17 00:00:00 2001 From: bruno Date: Sun, 8 Oct 2023 01:31:28 -0300 Subject: [PATCH 3/3] 78 - Fix --exclude parameter --- rasa_model_report/main.py | 1 - tests/test_main.py | 12 ++++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/rasa_model_report/main.py b/rasa_model_report/main.py index 47ae906..5371f0d 100644 --- a/rasa_model_report/main.py +++ b/rasa_model_report/main.py @@ -27,7 +27,6 @@ "-e", required=False, multiple=True, - type=list, help="List of utter and actions that will be exclude in the E2E test coverage. Use commas to separate items. " "Example: utter_greet,utter_goodbye,action_listen" ) diff --git a/tests/test_main.py b/tests/test_main.py index 4cb1075..6f7c59e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -12,7 +12,14 @@ def test_main_with_exclude(rasa_path): runner = CliRunner() result = runner.invoke( main, - ["--path", rasa_path, "--exclude", "utter_uncovered,utter_another_uncovered", "-e", "utter_goodbay"], + [ + "--path", + rasa_path, + "--exclude", + "utter_uncovered,utter_another_uncovered", + "-e", + "utter_goodbay", + ], ) assert os.path.isfile("model_report.md") is True assert utils.check_model_report_sections("model_report.md") is True @@ -31,7 +38,7 @@ def test_main_with_exclude(rasa_path): def test_main_with_valid_path(rasa_path): runner = CliRunner() result = runner.invoke(main, ["--path", rasa_path]) - assert os.path.isfile("model_report.md") + assert os.path.isfile("model_report.md") is True assert utils.check_model_report_sections("model_report.md") is True assert utils.check_model_report_images("model_report.md") is True assert result.exit_code == 0 @@ -62,5 +69,6 @@ def test_main_with_no_images(rasa_path): def test_main_help(): runner = CliRunner() result = runner.invoke(main, ["--help"]) + assert os.path.isfile("model_report.md") is False assert result.exit_code == 0 assert result.output