Merge pull request #85 from brunohjs/develop

New version `v1.4.1`

.github/workflows/test.yml

@@ -9,7 +9,7 @@ jobs:
     name: Lint and test
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
         os: [ubuntu-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.1] - 2023-10-09
+
+### Fixed
+- [#83](https://github.com/brunohjs/rasa-model-report/issues/83) Fix in element counting in `Element count` and `E2E coverage` section.
+
+
 ## [1.4.0] - 2023-10-08
 
 ### Added
@@ -89,6 +95,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - [#16](https://github.com/brunohjs/rasa-model-report/issues/16) Created a handler for retrieval intents in the report.
 
+
+[1.4.1]: https://github.com/brunohjs/rasa-model-report/compare/1.4.0...1.4.1
 [1.4.0]: https://github.com/brunohjs/rasa-model-report/compare/1.3.4...1.4.0
 [1.3.4]: https://github.com/brunohjs/rasa-model-report/compare/1.3.3...1.3.4
 [1.3.3]: https://github.com/brunohjs/rasa-model-report/compare/1.3.2...1.3.3

makefile

@@ -46,4 +46,4 @@ update-coverage:
 
 # Generate Rasa model report.
 report:
-	pip install . && rasa-model-report --path tests/mocks/rasa.v3/ --actions-path tests/mocks/rasa.v3/actions/ --rasa-version 3.5.2 --disable-nlu --no-images
+	pip install . && rasa-model-report --path tests/mocks/rasa.v3/ --actions-path tests/mocks/rasa.v3/actions/ --rasa-version 3.5.2 --disable-nlu --no-images --exclude utter_uncovered

rasa_model_report/controllers/e2e_coverage_controller.py

@@ -1,3 +1,4 @@
+import copy
 import glob
 import logging
 import re
@@ -20,7 +21,7 @@ def __init__(
         exclude: List[str],
         project_name: str,
         project_version: str,
-        **kwargs: Dict[str, Any]
+        **kwargs: Dict[str, Any],
     ):
         """
         __init__ method.
@@ -30,17 +31,29 @@ def __init__(
         :param project_name: Project name.
         :param project_version: Project version.
         """
-        super().__init__(rasa_path, output_path, project_name, project_version, actions_path=actions_path)
+        super().__init__(
+            rasa_path,
+            output_path,
+            project_name,
+            project_version,
+            actions_path=actions_path,
+        )
 
-        self._data: Dict[str, Union[List[str], float]] = {}
         self._total_num_elements: int = 0
         self._total_num_not_covered: int = 0
-        self._intents: List[str] = []
-        self._entities: List[str] = []
-        self._actions: List[str] = []
+        self._total_num_excluded: int = 0
+        self._items: Dict[str, Union[float, List[str]]] = {
+            item: [] for item in ["intents", "actions"]
+        }
+        self._not_covered_items: Dict[str, Union[float, List[str]]] = {
+            item: [] for item in ["intents", "actions"]
+        }
+        self._rate_items: Dict[str, Union[float, List[str]]] = {}
         self._total_rate: float = 0
-        self._exclude = exclude
-        self.json: JsonController = JsonController(rasa_path, output_path, project_name, project_version)
+        self._excluded_items = exclude
+        self.json: JsonController = JsonController(
+            rasa_path, output_path, project_name, project_version
+        )
 
         self._load_domain_elements()
         self._generate()
@@ -54,48 +67,51 @@ def _load_domain_elements(self) -> None:
         paths = [
             f"{self.nlu_path}/**/*.yml",
             f"{self.nlu_path}/*.yml",
-            f"{self.rasa_path}/*.yml"
+            f"{self.rasa_path}/*.yml",
         ]
         for path in paths:
             files.extend(glob.glob(path))
         for file in files:
             file_data = utils.load_yaml_file(file)
             if file_data:
-                for element in ["intents", "entities", "responses", "actions"]:
+                for element in ["intents", "responses", "actions"]:
                     data = []
                     for item in file_data.get(element, []):
                         if isinstance(item, str):
                             data.append(item)
                         else:
                             data.append(list(item.keys())[0])
                     if element == "responses":
-                        setattr(self, "_actions", getattr(self, "_actions") + data)
+                        self._items["actions"] += data
                     else:
-                        setattr(self, f"_{element}", getattr(self, f"_{element}") + data)
-        self._actions = list(dict.fromkeys(self._actions))
-        self._actions = self._exclude_special_actions()
-        self._actions = utils.list_diff(self._actions, self._exclude)
-        self._actions = utils.list_diff(self._actions, self.get_utters_in_actions())
+                        self._items[element] += data
+        self._items["actions"] = list(dict.fromkeys(self._items["actions"]))
+        self._update_not_covered_actions()
 
     def _generate(self) -> None:
         """
         Generate E2E tests coverage report string.
         """
-        not_covered = {}
         report_data = [item["name"] for item in self.json.core]
         for element in ["intents", "actions"]:
-            element_list = getattr(self, f"_{element}")
-            not_covered[element] = {
-                "items": utils.list_diff(element_list, report_data)
-            }
-            not_covered[element]["rate"] = 0
-            if element_list:
-                not_covered[element]["rate"] = 1 - len(not_covered[element]["items"]) / len(element_list)
-            self._total_num_elements += len(element_list)
-            self._total_num_not_covered += len(not_covered[element]["items"])
-        self._data = not_covered
-        if self._total_num_elements:
-            self._total_rate = 1 - self._total_num_not_covered / self._total_num_elements
+            self._not_covered_items[element] = utils.list_diff(
+                self._not_covered_items[element], report_data
+            )
+            self._rate_items[element] = 0
+            if self._items[element]:
+                self._rate_items[element] = 1 - len(
+                    self._not_covered_items[element]
+                ) / (
+                    len(self._items[element]) + self._total_num_excluded
+                    if element == "actions"
+                    else len(self._items[element])
+                )
+            self._total_num_elements += len(self._items[element])
+            self._total_num_not_covered += len(self._not_covered_items[element])
+        if self._total_num_elements - self._total_num_excluded:
+            self._total_rate = 1 - self._total_num_not_covered / (
+                self._total_num_elements - self._total_num_excluded
+            )
 
     def _exclude_special_actions(self) -> List[str]:
         """
@@ -104,14 +120,15 @@ def _exclude_special_actions(self) -> List[str]:
         :return: Actions list without special actions.
         """
         actions = []
-        for item in self._actions:
+        for item in self._items["actions"]:
             patterns_to_exclude = [
                 item.startswith("action_ask_"),
                 item.startswith("utter_ask_"),
-                item.startswith("validate_")
+                item.startswith("validate_"),
             ]
             if True not in patterns_to_exclude:
                 actions.append(item)
+        self._total_num_excluded += len(self._items["actions"]) - len(actions)
         return actions
 
     def get_utters_in_actions(self) -> List[str]:
@@ -121,18 +138,38 @@ def get_utters_in_actions(self) -> List[str]:
         :return: Found utter list.
         """
         result = []
-        pattern = r"(\"(utter|action)_[a-zA-Z0-9_-]+\")|((\'(utter|action)_[a-zA-Z0-9_-]+\'))"
-        actions_data = glob.glob(f"{self.actions_path}/**/*.py") + glob.glob(f"{self.actions_path}/*.py")
+        pattern = (
+            r"(\"(utter|action)_[a-zA-Z0-9_-]+\")|((\'(utter|action)_[a-zA-Z0-9_-]+\'))"
+        )
+        actions_data = glob.glob(f"{self.actions_path}/**/*.py") + glob.glob(
+            f"{self.actions_path}/*.py"
+        )
         actions_files = [open(file).read() for file in actions_data]
         for file in actions_files:
             strings = re.findall(pattern, file, re.UNICODE)
             if strings:
                 for string in strings:
-                    utter = string[0].replace("'", "").replace("\"", "")
+                    utter = string[0].replace("'", "").replace('"', "")
                     if utter not in result:
                         result.append(utter)
         return result
 
+    def _update_not_covered_actions(self) -> None:
+        """
+        Update not covered actions list.
+        """
+        self._not_covered_items = copy.deepcopy(self._items)
+        self._not_covered_items["actions"] = self._exclude_special_actions()
+        self._total_num_excluded += len(self._not_covered_items["actions"]) - len(
+            utils.list_diff(self._not_covered_items["actions"], self._excluded_items)
+        )
+        self._not_covered_items["actions"] = utils.list_diff(
+            self._not_covered_items["actions"], self._excluded_items
+        )
+        self._not_covered_items["actions"] = utils.list_diff(
+            self._not_covered_items["actions"], self.get_utters_in_actions()
+        )
+
     def save(self) -> None:
         """
         Save E2E tests coverage report to a text file.
@@ -144,17 +181,22 @@ def save(self) -> None:
                 file.write("End-to-end tests coverage report\n")
                 file.write("-------------------------------------------------\n")
                 file.write("Elements that aren't covered:\n")
-                for element in self._data:
+                for element in self._not_covered_items:
                     file.write(f" + {element.capitalize()}\n")
-                    if self._data[element].get("items"):
-                        for item in self._data[element]["items"]:
+                    if self._not_covered_items[element]:
+                        for item in self._not_covered_items[element]:
                             file.write(f"   - {item}\n")
                     else:
                         file.write("   - (no elements not covered)\n")
                 file.write("\n")
-                file.write(f"Total number of elements: {self._total_num_elements}\n")
-                file.write(f"Total number of not covered elements: {self._total_num_not_covered}\n")
-                file.write(f"Coverage rate: {self._total_rate * 100:.1f}%\n")
+                file.write(f"Total number of elements: {self.total_num_elements}\n")
+                file.write(
+                    f"Total number of not covered elements: {self.total_num_not_covered}\n"
+                )
+                file.write(
+                    f"Total number of excluded elements: {self.total_num_excluded}\n"
+                )
+                file.write(f"Coverage rate: {self.total_rate * 100:.1f}%\n")
                 file.write("-------------------------------------------------\n")
             logging.info(f"{file_path} file successfully saved.")
         else:
@@ -169,13 +211,22 @@ def have_not_covered_items(self) -> bool:
         return self._total_num_not_covered > 0
 
     @property
-    def data(self) -> Dict[str, Union[List[str], float]]:
+    def items(self) -> List[str]:
+        """
+        Get E2E tests elements data.
+
+        :return: Copy of E2E tests elements data.
+        """
+        return copy.deepcopy(self._items)
+
+    @property
+    def not_covered_items(self) -> Dict[str, Union[List[str], float]]:
         """
         Get E2E tests not covered elements data.
 
         :return: Copy of E2E tests not covered elements data.
         """
-        return self._data.copy()
+        return copy.deepcopy(self._not_covered_items)
 
     @property
     def total_rate(self) -> float:
@@ -187,13 +238,17 @@ def total_rate(self) -> float:
         return self._total_rate
 
     @property
-    def total_num_elements(self) -> int:
+    def total_num_elements(self, with_excluded_items: bool = False) -> int:
         """
         Get total number of E2E tests elements.
 
         :return int: Total number of E2E tests elements.
         """
-        return self._total_num_elements
+        return (
+            self._total_num_elements
+            if with_excluded_items
+            else self._total_num_elements - self._total_num_excluded
+        )
 
     @property
     def total_num_not_covered(self) -> int:
@@ -203,3 +258,12 @@ def total_num_not_covered(self) -> int:
         :return int: Total number of E2E tests not covered elements.
         """
         return self._total_num_not_covered
+
+    @property
+    def total_num_excluded(self) -> int:
+        """
+        Get total number of E2E tests excluded elements.
+
+        :return int: Total number of E2E tests excluded elements.
+        """
+        return self._total_num_excluded

rasa_model_report/controllers/markdown_controller.py

@@ -149,7 +149,7 @@ def build_table(self, data: List[Union[str, float]]) -> str:
         :return: Table in markdown format.
         """
         header = f"|#|{'|'.join(data[0])}|\n"
-        header += f"{'|-' * (len(data[0]) + 1)}|\n"
+        header += f"|:-:{'|-' * len(data[0])}|\n"
         content = ""
         for index, row in enumerate(data[1:]):
             text_row = f"|{index + 1}|{'|'.join(row)}|\n"
@@ -227,7 +227,7 @@ def build_element_count(self) -> str:
         """
         intents = len(self.json.intents)
         entities = len(self.json.entities)
-        utters_actions = self.e2e_coverage.total_num_elements
+        utters_actions = len(self.e2e_coverage.items["actions"])
         stories_rules = utils.count_stories_and_rules(self.rasa_path)
         stories = stories_rules.get("stories")
         rules = stories_rules.get("rules")
@@ -606,7 +606,7 @@ def build_e2e_coverage_title(self) -> str:
         :return: Title block in markdown format.
         """
         title = "## E2E Coverage <a name='e2e'></a>\n"
-        description = "Section that shows data from intents, entities " \
+        description = "Section that shows data from intents " \
             "and responses that aren't covered by end-to-end tests.\n"
         return title + description
 
@@ -620,21 +620,23 @@ def build_e2e_coverage_list(self) -> str:
         description = "List with not covered elements by end-to-end tests.\n"
         title += description + "\n"
         text = ""
-        data = self.e2e_coverage.data
+        data = self.e2e_coverage.not_covered_items
         rate = self.e2e_coverage.total_rate
         total_num_elements = self.e2e_coverage.total_num_elements
         total_num_not_covered = self.e2e_coverage.total_num_not_covered
+        total_num_excluded = self.e2e_coverage.total_num_excluded
         if data:
-            for element in data:
+            for element in ["intents", "actions"]:
                 text += f"#### {element.capitalize()}\n"
-                if data[element]["items"]:
-                    for item in data[element]["items"]:
+                if data[element]:
+                    for item in data[element]:
                         text += f" - {item}\n"
                 else:
                     text += " - (no elements not covered)\n"
                 text += "\n"
             text += f"Total number of elements: {total_num_elements}\n\n"
             text += f"Total number of not covered elements: {total_num_not_covered}\n\n"
+            text += f"Total number of excluded elements: {total_num_excluded}\n\n"
             text += f"Coverage rate: {rate * 100:.1f}% ({utils.get_color(rate)})\n\n"
         else:
             text = "\nThere are no end-to-end tests coverage.\n"

tests/mocks/rasa.v2/domain.yml

@@ -15,6 +15,7 @@ entities:
 
 actions:
   - utter_greet
+  - action_hello_world
 
 responses:
   utter_greet:

tests/mocks/rasa.v3/domain.yml

@@ -14,6 +14,10 @@ intents:
 entities:
   - day
 
+actions:
+  - utter_greet
+  - action_hello_world
+
 responses:
   utter_greet:
   - text: "Hey! How are you?"