Merge pull request #79 from brunohjs/#78

78 - Parameter `--exclude` to exclude utters and actions

CHANGELOG.md

@@ -5,10 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [1.3.5] - In progress...
+## [1.4.0] - In progress...
 
 ### Added
 - [#73](https://github.com/brunohjs/rasa-model-report/issues/73) Created section with element count on model report. Additionally, now all tables elements are numbered.
+- [#78](https://github.com/brunohjs/rasa-model-report/issues/78) Created `--exclude` CLI command parameter. This command is used to exclude utters and actions from end-to-end test coverage.
 
 ### Fixed
 - [#75](https://github.com/brunohjs/rasa-model-report/issues/75) Fixed bug with utters in actions that weren't being covered by the end-to-end tests.
@@ -88,7 +89,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - [#16](https://github.com/brunohjs/rasa-model-report/issues/16) Created a handler for retrieval intents in the report.
 
-[1.3.5]: https://github.com/brunohjs/rasa-model-report/compare/1.3.4...1.3.5
+[1.3.5]: https://github.com/brunohjs/rasa-model-report/compare/1.3.4...1.4.0
 [1.3.4]: https://github.com/brunohjs/rasa-model-report/compare/1.3.3...1.3.4
 [1.3.3]: https://github.com/brunohjs/rasa-model-report/compare/1.3.2...1.3.3
 [1.3.2]: https://github.com/brunohjs/rasa-model-report/compare/1.3.0...1.3.2

README.md

@@ -93,6 +93,9 @@ There are parameters that can be used. Available options are below:
                         path)
 --disable-nlu           Disable processing NLU sentences. NLU section will
                         not be generated in the report. Required Rasa API.
+-e, --exclude LIST      List of utter and actions that will be exclude in
+                        the E2E test coverage. Use commas to separate items.
+                        Example: utter_greet,utter_goodbye,action_listen
 -h, --help              Show this help message.
 --model-link TEXT       Model download link. It's only displayed in the
                         report to model download.

rasa_model_report/controllers/e2e_coverage_controller.py

@@ -17,6 +17,7 @@ def __init__(
         rasa_path: str,
         output_path: str,
         actions_path: str,
+        exclude: List[str],
         project_name: str,
         project_version: str,
         **kwargs: Dict[str, Any]
@@ -38,6 +39,7 @@ def __init__(
         self._entities: List[str] = []
         self._actions: List[str] = []
         self._total_rate: float = 0
+        self._exclude = exclude
         self.json: JsonController = JsonController(rasa_path, output_path, project_name, project_version)
 
         self._load_domain_elements()
@@ -72,6 +74,7 @@ def _load_domain_elements(self) -> None:
                         setattr(self, f"_{element}", getattr(self, f"_{element}") + data)
         self._actions = list(dict.fromkeys(self._actions))
         self._actions = self._exclude_special_actions()
+        self._actions = utils.list_diff(self._actions, self._exclude)
         self._actions = utils.list_diff(self._actions, self.get_utters_in_actions())
 
     def _generate(self) -> None:

rasa_model_report/controllers/markdown_controller.py

@@ -61,6 +61,7 @@ def __init__(
             rasa_path,
             output_path,
             kwargs.get("actions_path"),
+            kwargs.get("exclude", []),
             project_name,
             project_version
         )

rasa_model_report/helpers/constants.py

@@ -7,3 +7,4 @@
 RASA_API_URL = "http://localhost:5005"
 RASA_PATH = "./"
 RASA_VERSION = None
+EXCLUDE = []

rasa_model_report/main.py

@@ -22,6 +22,14 @@
     help="Disable processing NLU sentences. NLU section will not be generated "
     "in the report. Required Rasa API."
 )
+@click.option(
+    "--exclude",
+    "-e",
+    required=False,
+    multiple=True,
+    help="List of utter and actions that will be exclude in the E2E test coverage. Use commas to separate items. "
+    "Example: utter_greet,utter_goodbye,action_listen"
+)
 @click.help_option(
     "--help",
     "-h",
@@ -101,6 +109,7 @@
 def main(
     actions_path,
     disable_nlu,
+    exclude,
     model_link,
     no_images,
     output_path, path,
@@ -124,6 +133,7 @@ def main(
         model_link=model_link,
         actions_path=actions_path,
         no_images=no_images,
-        precision=precision
+        precision=precision,
+        exclude=[item for row in exclude for item in row.split(",")]
     )
     return report

tests/conftest.py

@@ -14,8 +14,8 @@
 
 def pytest_generate_tests(metafunc):
     rasa_dirs = glob.glob("tests/mocks/rasa.v*")
-    metafunc.fixturenames.append('rasa_path')
-    metafunc.parametrize('rasa_path', rasa_dirs)
+    metafunc.fixturenames.append("rasa_path")
+    metafunc.parametrize("rasa_path", rasa_dirs)
 
 
 @pytest.fixture(autouse=True)
@@ -33,13 +33,24 @@ def load_controllers(rasa_path):
     project_name = "test-project"
     project_version = "0.0.0"
     rasa_version = "0.0.0"
+    exclude = []
     utils.load_mock_payloads()
     controller = Controller(rasa_path, output_path, project_name, project_version)
-    json_controller = JsonController(rasa_path, output_path, project_name, project_version)
-    csv_controller = CsvController(rasa_path, output_path, project_name, project_version)
-    nlu_controller = NluController(rasa_path, output_path, project_name, project_version)
-    e2e_coverage_controller = E2ECoverageController(rasa_path, output_path, actions_path, project_name, project_version)
-    markdown_controller = MarkdownController(rasa_path, output_path, project_name, rasa_version, project_version)
+    json_controller = JsonController(
+        rasa_path, output_path, project_name, project_version
+    )
+    csv_controller = CsvController(
+        rasa_path, output_path, project_name, project_version
+    )
+    nlu_controller = NluController(
+        rasa_path, output_path, project_name, project_version
+    )
+    e2e_coverage_controller = E2ECoverageController(
+        rasa_path, output_path, actions_path, exclude, project_name, project_version
+    )
+    markdown_controller = MarkdownController(
+        rasa_path, output_path, project_name, rasa_version, project_version
+    )
     pytest.controller = controller
     pytest.json_controller = json_controller
     pytest.csv_controller = csv_controller

tests/mocks/rasa.v2/domain.yml

@@ -42,6 +42,12 @@ responses:
   utter_test_2:
   - text: "Test 2"
 
+  utter_uncovered:
+  - text: "Uncovered utter"
+
+  utter_another_uncovered:
+  - text: "Another uncovered utter"
+
 session_config:
   session_expiration_time: 60
   carry_over_slots_to_new_session: true

tests/mocks/rasa.v3/domain.yml

@@ -40,6 +40,12 @@ responses:
   utter_test_2:
   - text: "Test 2"
 
+  utter_uncovered:
+  - text: "Uncovered utter"
+
+  utter_another_uncovered:
+  - text: "Another uncovered utter"
+
 session_config:
   session_expiration_time: 60
   carry_over_slots_to_new_session: true

tests/test_e2e_coverage_controller.py

@@ -12,8 +12,9 @@ def execute_before_each_test(rasa_path):
     actions_path = None
     project_name = "test-project"
     project_version = "0.0.0"
+    exclude = []
     e2e_coverage_controller = E2ECoverageController(
-        rasa_path, output_path, actions_path, project_name, project_version
+        rasa_path, output_path, actions_path, exclude, project_name, project_version
     )
     pytest.e2e_coverage_controller = e2e_coverage_controller
     yield

tests/test_main.py

@@ -7,13 +7,40 @@
 from tests import utils
 
 
+@responses.activate
+def test_main_with_exclude(rasa_path):
+    runner = CliRunner()
+    result = runner.invoke(
+        main,
+        [
+            "--path",
+            rasa_path,
+            "--exclude",
+            "utter_uncovered,utter_another_uncovered",
+            "-e",
+            "utter_goodbay",
+        ],
+    )
+    assert os.path.isfile("model_report.md") is True
+    assert utils.check_model_report_sections("model_report.md") is True
+    assert utils.check_model_report_images("model_report.md") is True
+    assert (
+        utils.check_model_report_text(
+            "model_report.md", ["utter_uncovered", "utter_another_uncovered"]
+        )
+        is False
+    )
+    assert result.exit_code == 0
+    assert result.output == ""
+
+
 @responses.activate
 def test_main_with_valid_path(rasa_path):
     runner = CliRunner()
     result = runner.invoke(main, ["--path", rasa_path])
-    assert os.path.isfile("model_report.md")
-    assert utils.check_model_report_sections("model_report.md")
-    assert utils.check_model_report_images("model_report.md")
+    assert os.path.isfile("model_report.md") is True
+    assert utils.check_model_report_sections("model_report.md") is True
+    assert utils.check_model_report_images("model_report.md") is True
     assert result.exit_code == 0
     assert result.output == ""
 
@@ -23,7 +50,7 @@ def test_main_with_invalid_path():
     utils.load_mock_payloads()
     runner = CliRunner()
     result = runner.invoke(main, [])
-    assert not os.path.isfile("model_report.md")
+    assert os.path.isfile("model_report.md") is False
     assert result.exit_code == 0
     assert result.output == ""
 
@@ -33,14 +60,15 @@ def test_main_with_no_images(rasa_path):
     utils.load_mock_payloads()
     runner = CliRunner()
     result = runner.invoke(main, ["--path", rasa_path, "--no-images"])
-    assert os.path.isfile("model_report.md")
-    assert not utils.check_model_report_images("model_report.md")
+    assert os.path.isfile("model_report.md") is True
+    assert utils.check_model_report_images("model_report.md") is False
     assert result.exit_code == 0
     assert result.output == ""
 
 
 def test_main_help():
     runner = CliRunner()
     result = runner.invoke(main, ["--help"])
+    assert os.path.isfile("model_report.md") is False
     assert result.exit_code == 0
     assert result.output

tests/utils.py

@@ -77,3 +77,13 @@ def check_model_report_images(model_report_path):
         "/DIETClassifier_confusion_matrix.png" in file_data and \
         "/story_confusion_matrix.png" in file_data and \
         "/intent_histogram.png" in file_data
+
+
+def check_model_report_text(model_report_path, texts):
+    file = open(model_report_path, encoding="utf-8")
+    file_data = file.read()
+    file.close()
+    if isinstance(texts, str):
+        return texts in file_data
+    else:
+        return all([text in file_data for text in texts])