Merge pull request #1882 from RasaHQ/e2e-eval-fix

Parse intent messages using RegexInterpreter

CHANGELOG.rst

@@ -4,6 +4,13 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to `Semantic Versioning`_ starting with version 0.2.0.
 
+[0.13.6] - 2019-03-28
+^^^^^^^^^^^^^^^^^^^^^
+
+Fixed
+-----
+- correctly process intent messages in end-to-end evaluations
+
 [0.13.4] - 2019-03-19
 ^^^^^^^^^^^^^^^^^^^^^
 

rasa_core/evaluate.py

@@ -1,16 +1,16 @@
-from collections import defaultdict
-from collections import namedtuple
-
 import argparse
 import io
 import json
 import logging
-import numpy as np
 import os
 import warnings
+from collections import defaultdict
+from collections import namedtuple
+from typing import List, Optional, Any, Text, Dict, Tuple
+
+import numpy as np
 from sklearn.exceptions import UndefinedMetricWarning
 from tqdm import tqdm
-from typing import List, Optional, Any, Text, Dict, Tuple
 
 from rasa_core import training, cli
 from rasa_core import utils
@@ -99,13 +99,13 @@ class EvaluationStore(object):
     """Class storing action, intent and entity predictions and targets."""
 
     def __init__(
-        self,
-        action_predictions: Optional[List[str]] = None,
-        action_targets: Optional[List[str]] = None,
-        intent_predictions: Optional[List[str]] = None,
-        intent_targets: Optional[List[str]] = None,
-        entity_predictions: Optional[List[Dict[Text, Any]]] = None,
-        entity_targets: Optional[List[Dict[Text, Any]]] = None
+            self,
+            action_predictions: Optional[List[str]] = None,
+            action_targets: Optional[List[str]] = None,
+            intent_predictions: Optional[List[str]] = None,
+            intent_targets: Optional[List[str]] = None,
+            entity_predictions: Optional[List[Dict[Text, Any]]] = None,
+            entity_targets: Optional[List[Dict[Text, Any]]] = None
     ) -> None:
         self.action_predictions = action_predictions or []
         self.action_targets = action_targets or []
@@ -115,13 +115,13 @@ def __init__(
         self.entity_targets = entity_targets or []
 
     def add_to_store(
-        self,
-        action_predictions: Optional[List[str]] = None,
-        action_targets: Optional[List[str]] = None,
-        intent_predictions: Optional[List[str]] = None,
-        intent_targets: Optional[List[str]] = None,
-        entity_predictions: Optional[List[Dict[Text, Any]]] = None,
-        entity_targets: Optional[List[Dict[Text, Any]]] = None
+            self,
+            action_predictions: Optional[List[str]] = None,
+            action_targets: Optional[List[str]] = None,
+            intent_predictions: Optional[List[str]] = None,
+            intent_targets: Optional[List[str]] = None,
+            entity_predictions: Optional[List[Dict[Text, Any]]] = None,
+            entity_targets: Optional[List[Dict[Text, Any]]] = None
     ) -> None:
         """Add items or lists of items to the store"""
         for k, v in locals().items():
@@ -399,10 +399,10 @@ def _in_training_data_fraction(action_list):
 
 
 def collect_story_predictions(
-    completed_trackers: List[DialogueStateTracker],
-    agent: Agent,
-    fail_on_prediction_errors: bool = False,
-    use_e2e: bool = False
+        completed_trackers: List[DialogueStateTracker],
+        agent: Agent,
+        fail_on_prediction_errors: bool = False,
+        use_e2e: bool = False
 ) -> Tuple[StoryEvalution, int]:
     """Test the stories from a file, running them through the stored model."""
 

rasa_core/training/dsl.py

@@ -8,6 +8,7 @@
 from typing import Optional, List, Text, Any, Dict, AnyStr
 
 from rasa_core import utils
+from rasa_core.constants import INTENT_MESSAGE_PREFIX
 from rasa_core.events import (
     ActionExecuted, UserUttered, Event, SlotSet)
 from rasa_core.exceptions import StoryParseError
@@ -326,7 +327,10 @@ def add_checkpoint(self,
         self.current_step_builder.add_checkpoint(name, conditions)
 
     def _parse_message(self, message, line_num):
-        parse_data = self.interpreter.parse(message)
+        if message.startswith(INTENT_MESSAGE_PREFIX):
+            parse_data = RegexInterpreter().parse(message)
+        else:
+            parse_data = self.interpreter.parse(message)
         utterance = UserUttered(message,
                                 parse_data.get("intent"),
                                 parse_data.get("entities"),

rasa_core/version.py

@@ -1 +1 @@
-__version__ = '0.13.5'
+__version__ = '0.13.6'