From b733e0fe2b0db2838ee7903613a2968c85fc16a9 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Wed, 17 Jul 2019 17:11:52 +0200 Subject: [PATCH 01/22] Edit serialisation, change WronglyClassifiedUserUtterance class initialisation --- rasa/core/test.py | 105 +++++++++++++++++---------------------------- rasa/core/utils.py | 5 --- 2 files changed, 40 insertions(+), 70 deletions(-) diff --git a/rasa/core/test.py b/rasa/core/test.py index e1b523bf55a2..816b13d1973f 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional, Text, Tuple from rasa.constants import RESULTS_FILE -from rasa.core.events import ActionExecuted, UserUttered +from rasa.core.events import ActionExecuted, UserUttered, Event if typing.TYPE_CHECKING: from rasa.core.agent import Agent @@ -76,32 +76,25 @@ def has_prediction_target_mismatch(self): or self.action_predictions != self.action_targets ) - def serialise_targets( - self, include_actions=True, include_intents=True, include_entities=False - ): - targets = [] - if include_actions: - targets += self.action_targets - if include_intents: - targets += self.intent_targets - if include_entities: - targets += self.entity_targets - - return [json.dumps(t) if isinstance(t, dict) else t for t in targets] - - def serialise_predictions( - self, include_actions=True, include_intents=True, include_entities=False - ): - predictions = [] + def serialise(self): + """Turn targets and predictions to lists of equal size for sklearn""" + + targets = ( + self.action_targets + + self.intent_targets + + [json.dumps(t) for t in self.entity_targets] + ) + predictions = ( + self.action_predictions + + self.intent_predictions + + [json.dumps(p) for p in self.entity_predictions] + ) - if include_actions: - predictions += self.action_predictions - if include_intents: - predictions += self.intent_predictions - if include_entities: - predictions += self.entity_predictions + # sklearn does not cope with lists of unequal size, nor None values + padding = len(targets) - len(predictions) + predictions += ["None"] * padding - return [json.dumps(t) if isinstance(t, dict) else t for t in predictions] + return targets, predictions class WronglyPredictedAction(ActionExecuted): @@ -144,24 +137,23 @@ class WronglyClassifiedUserUtterance(UserUttered): type_name = "wrong_utterance" - def __init__( - self, - text, - correct_intent, - correct_entities, - parse_data=None, - timestamp=None, - input_channel=None, - predicted_intent=None, - predicted_entities=None, - ): - self.predicted_intent = predicted_intent - self.predicted_entities = predicted_entities + def __init__(self, event: UserUttered, eval_store: EvaluationStore): + + if eval_store.intent_predictions == list(): + self.predicted_intent = None + else: + self.predicted_intent = eval_store.intent_predictions[0] + self.predicted_entities = eval_store.entity_predictions - intent = {"name": correct_intent} + intent = {"name": eval_store.intent_targets[0]} super(WronglyClassifiedUserUtterance, self).__init__( - text, intent, correct_entities, parse_data, timestamp, input_channel + event.text, + intent, + eval_store.entity_targets, + event.parse_data, + event.timestamp, + event.input_channel, ) def as_story_string(self, e2e=True): @@ -207,14 +199,14 @@ def _clean_entity_results(entity_results): def _collect_user_uttered_predictions( event, partial_tracker, fail_on_prediction_errors ): - from rasa.core.utils import pad_list_to_size - user_uttered_eval_store = EvaluationStore() intent_gold = event.parse_data.get("true_intent") predicted_intent = event.parse_data.get("intent").get("name") - if predicted_intent is None: - predicted_intent = "None" + + if predicted_intent == list(): + predicted_intent = [None] + user_uttered_eval_store.add_to_store( intent_predictions=predicted_intent, intent_targets=intent_gold ) @@ -223,13 +215,6 @@ def _collect_user_uttered_predictions( predicted_entities = event.parse_data.get("entities") if entity_gold or predicted_entities: - if len(entity_gold) > len(predicted_entities): - predicted_entities = pad_list_to_size( - predicted_entities, len(entity_gold), "None" - ) - elif len(predicted_entities) > len(entity_gold): - entity_gold = pad_list_to_size(entity_gold, len(predicted_entities), "None") - user_uttered_eval_store.add_to_store( entity_targets=_clean_entity_results(entity_gold), entity_predictions=_clean_entity_results(predicted_entities), @@ -237,16 +222,7 @@ def _collect_user_uttered_predictions( if user_uttered_eval_store.has_prediction_target_mismatch(): partial_tracker.update( - WronglyClassifiedUserUtterance( - event.text, - intent_gold, - user_uttered_eval_store.entity_predictions, - event.parse_data, - event.timestamp, - event.input_channel, - predicted_intent, - user_uttered_eval_store.entity_targets, - ) + WronglyClassifiedUserUtterance(event, user_uttered_eval_store) ) if fail_on_prediction_errors: raise ValueError( @@ -493,10 +469,9 @@ async def test( from sklearn.exceptions import UndefinedMetricWarning warnings.simplefilter("ignore", UndefinedMetricWarning) - report, precision, f1, accuracy = get_evaluation_metrics( - evaluation_store.serialise_targets(), - evaluation_store.serialise_predictions(), - ) + + targets, predictions = evaluation_store.serialise() + report, precision, f1, accuracy = get_evaluation_metrics(targets, predictions) if out_directory: plot_story_evaluation( diff --git a/rasa/core/utils.py b/rasa/core/utils.py index ac8691b83610..23dcf18080f2 100644 --- a/rasa/core/utils.py +++ b/rasa/core/utils.py @@ -376,11 +376,6 @@ def remove_none_values(obj: Dict[Text, Any]) -> Dict[Text, Any]: return {k: v for k, v in obj.items() if v is not None} -def pad_list_to_size(_list, size, padding_value=None): - """Pads _list with padding_value up to size""" - return _list + [padding_value] * (size - len(_list)) - - class AvailableEndpoints(object): """Collection of configured endpoints.""" From 89446a7aa91fd0cdd32f46ab008b18f9d587005c Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Wed, 17 Jul 2019 17:12:52 +0200 Subject: [PATCH 02/22] Update CHANGELOG.rst --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 745b532e77c1..c4e7d7b4d94a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -25,6 +25,7 @@ Removed Fixed ----- - validation no longer throws an error during interactive learning +- messages with multiple entities are now handled properly with e2e evaluation [1.1.6] - 2019-07-12 ^^^^^^^^^^^^^^^^^^^^ From de9617e7d68f39f9b0ad28ebac6c6b75c8b81d00 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Thu, 18 Jul 2019 15:10:25 +0200 Subject: [PATCH 03/22] Use restaurantbot for e2e testing --- CHANGELOG.rst | 3 +- data/test_evaluations/end_to_end_story.md | 38 +++++--- examples/restaurantbot/README.md | 6 -- examples/restaurantbot/run.py | 114 ---------------------- rasa/core/test.py | 2 +- tests/core/conftest.py | 7 ++ tests/core/test_evaluation.py | 64 ++++++++---- tests/core/test_utils.py | 9 -- 8 files changed, 81 insertions(+), 162 deletions(-) delete mode 100644 examples/restaurantbot/run.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c4e7d7b4d94a..d64aaeb582c7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,7 +16,8 @@ Added Changed ------- - +- restaurantbot model was replaced with a stacked model +- ``end_to_end_story.md`` was re-written in the restaurantbot domain Removed ------- diff --git a/data/test_evaluations/end_to_end_story.md b/data/test_evaluations/end_to_end_story.md index 93f91684e6f8..0600e9a9da41 100644 --- a/data/test_evaluations/end_to_end_story.md +++ b/data/test_evaluations/end_to_end_story.md @@ -1,17 +1,27 @@ ## simple_story_with_only_start -> check_greet -* default:/default - - utter_default - -## simple_story_with_only_end -* greet:/greet - - utter_greet -> check_greet +* greet: Hello + - utter_ask_howcanhelp ## simple_story_with_multiple_turns -* greet:/greet - - utter_greet -* default:/default - - utter_default -* goodbye:/goodbye - - utter_goodbye +* greet: good morning + - utter_ask_howcanhelp +* inform: im looking for a [moderately](price:moderate) priced restaurant in the [east](location) part of town + - utter_on_it + - utter_ask_cuisine +* inform: [french](cuisine) food + - utter_ask_numpeople + + ## story_with_multiple_entities_correction_and_search +* greet: hello + - utter_ask_howcanhelp +* inform: im looking for a [cheap](price:lo) restaurant which has [french](cuisine) food and is located in [bombay](location) + - utter_on_it + - utter_ask_numpeople +* inform: for [six](people:6) please + - utter_ask_moreupdates +* inform: actually i need a [moderately](price:moderate) priced restaurant + - utter_ask_moreupdates +* deny: no + - utter_ack_dosearch + - action_search_restaurants + - action_suggest \ No newline at end of file diff --git a/examples/restaurantbot/README.md b/examples/restaurantbot/README.md index da522c27ce03..18824f615fff 100644 --- a/examples/restaurantbot/README.md +++ b/examples/restaurantbot/README.md @@ -1,10 +1,5 @@ # Restaurant Bot -This example includes a file called `run.py`, which contains an example -of how to use Rasa directly from your python code. - -## What’s inside this example? - This example contains some training data and the main files needed to build an assistant on your local machine. The `restaurantbot` consists of the following files: @@ -15,7 +10,6 @@ assistant on your local machine. The `restaurantbot` consists of the following f - **domain.yml** contains the domain of the assistant - **endpoints.yml** contains the webhook configuration for the custom action - **policy.py** contains a custom policy -- **run.py** contains code to train a Rasa model and use it to parse some text ## How to use this example? diff --git a/examples/restaurantbot/run.py b/examples/restaurantbot/run.py deleted file mode 100644 index 2add336a84ca..000000000000 --- a/examples/restaurantbot/run.py +++ /dev/null @@ -1,114 +0,0 @@ -import argparse -import asyncio -import logging -from typing import Text - -import os -import rasa.utils.io -import rasa.train -from examples.restaurantbot.policy import RestaurantPolicy -from rasa.core.agent import Agent -from rasa.core.policies.memoization import MemoizationPolicy -from rasa.core.policies.mapping_policy import MappingPolicy - -logger = logging.getLogger(__name__) - - -async def parse(text: Text, model_path: Text): - agent = Agent.load(model_path) - - response = await agent.handle_text(text) - - logger.info("Text: '{}'".format(text)) - logger.info("Response:") - logger.info(response) - - return response - - -async def train_core( - domain_file: Text = "domain.yml", - model_directory: Text = "models", - model_name: Text = "current", - training_data_file: Text = "data/stories.md", -): - agent = Agent( - domain_file, - policies=[ - MemoizationPolicy(max_history=3), - MappingPolicy(), - RestaurantPolicy(batch_size=100, epochs=400, validation_split=0.2), - ], - ) - - training_data = await agent.load_data(training_data_file) - agent.train(training_data) - - # Attention: agent.persist stores the model and all meta data into a folder. - # The folder itself is not zipped. - model_path = os.path.join(model_directory, model_name, "core") - agent.persist(model_path) - - logger.info("Model trained. Stored in '{}'.".format(model_path)) - - return model_path - - -def train_nlu( - config_file="config.yml", - model_directory: Text = "models", - model_name: Text = "current", - training_data_file="data/nlu.md", -): - from rasa.nlu.training_data import load_data - from rasa.nlu import config - from rasa.nlu.model import Trainer - - training_data = load_data(training_data_file) - trainer = Trainer(config.load(config_file)) - trainer.train(training_data) - - # Attention: trainer.persist stores the model and all meta data into a folder. - # The folder itself is not zipped. - model_path = os.path.join(model_directory, model_name) - model_directory = trainer.persist(model_path, fixed_model_name="nlu") - - logger.info("Model trained. Stored in '{}'.".format(model_directory)) - - return model_directory - - -if __name__ == "__main__": - rasa.utils.io.configure_colored_logging(loglevel="INFO") - - parser = argparse.ArgumentParser(description="Restaurant Bot") - - subparser = parser.add_subparsers(dest="subparser_name") - train_parser = subparser.add_parser("train", help="train a core or nlu model") - parse_parser = subparser.add_parser("predict", help="predict next action") - - parse_parser.add_argument( - "--model", - default="models/current", - help="Path to the model directory which contains " - "sub-folders for core and nlu models.", - ) - parse_parser.add_argument("--text", default="hello", help="Text to parse.") - - train_parser.add_argument( - "model", - choices=["nlu", "core"], - help="Do you want to train a NLU or Core model?", - ) - args = parser.parse_args() - - loop = asyncio.get_event_loop() - - # decide what to do based on first parameter of the script - if args.subparser_name == "train": - if args.model == "nlu": - train_nlu() - elif args.model == "core": - loop.run_until_complete(train_core()) - elif args.subparser_name == "predict": - loop.run_until_complete(parse(args.text, args.model)) diff --git a/rasa/core/test.py b/rasa/core/test.py index 816b13d1973f..1e53cae5d361 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional, Text, Tuple from rasa.constants import RESULTS_FILE -from rasa.core.events import ActionExecuted, UserUttered, Event +from rasa.core.events import ActionExecuted, UserUttered if typing.TYPE_CHECKING: from rasa.core.agent import Agent diff --git a/tests/core/conftest.py b/tests/core/conftest.py index fd023db2f5b5..bf5d947783e3 100644 --- a/tests/core/conftest.py +++ b/tests/core/conftest.py @@ -43,6 +43,8 @@ MOODBOT_MODEL_PATH = "examples/moodbot/models/" +RESTAURANTBOT_MODEL_PATH = "examples/restaurantbot/models/" + DEFAULT_ENDPOINTS_FILE = "data/test_endpoints/example_endpoints.yml" TEST_DIALOGUES = [ @@ -235,3 +237,8 @@ def train_model(project: Text, filename: Text = "test.tar.gz"): @pytest.fixture(scope="session") def trained_model(project) -> Text: return train_model(project) + + +@pytest.fixture +def restaurantbot() -> Agent: + return Agent.load_local_model(RESTAURANTBOT_MODEL_PATH) diff --git a/tests/core/test_evaluation.py b/tests/core/test_evaluation.py index d2a042a2701a..6e51851045a4 100644 --- a/tests/core/test_evaluation.py +++ b/tests/core/test_evaluation.py @@ -28,31 +28,61 @@ async def test_evaluation_image_creation(tmpdir, default_agent): assert os.path.isfile(stories_path) -async def test_action_evaluation_script(tmpdir, default_agent): +async def test_end_to_end_evaluation_script(tmpdir, restaurantbot): completed_trackers = await _generate_trackers( - DEFAULT_STORIES_FILE, default_agent, use_e2e=False - ) - story_evaluation, num_stories = collect_story_predictions( - completed_trackers, default_agent, use_e2e=False - ) - - assert not story_evaluation.evaluation_store.has_prediction_target_mismatch() - assert len(story_evaluation.failed_stories) == 0 - assert num_stories == 3 - - -async def test_end_to_end_evaluation_script(tmpdir, default_agent): - completed_trackers = await _generate_trackers( - END_TO_END_STORY_FILE, default_agent, use_e2e=True + END_TO_END_STORY_FILE, restaurantbot, use_e2e=True ) story_evaluation, num_stories = collect_story_predictions( - completed_trackers, default_agent, use_e2e=True + completed_trackers, restaurantbot, use_e2e=True ) + serialised_actions = [ + "utter_ask_howcanhelp", + "action_listen", + "utter_ask_howcanhelp", + "action_listen", + "utter_on_it", + "utter_ask_cuisine", + "action_listen", + "utter_ask_numpeople", + "action_listen", + "utter_ask_howcanhelp", + "action_listen", + "utter_on_it", + "utter_ask_numpeople", + "action_listen", + "utter_ask_moreupdates", + "action_listen", + "utter_ask_moreupdates", + "action_listen", + "utter_ack_dosearch", + "action_search_restaurants", + "action_suggest", + "action_listen", + "greet", + "greet", + "inform", + "inform", + "greet", + "inform", + "inform", + "inform", + "deny", + '{"start": 17, "end": 27, "entity": "price", "value": "moderate"}', + '{"start": 53, "end": 57, "entity": "location", "value": "east"}', + '{"start": 0, "end": 6, "entity": "cuisine", "value": "french"}', + '{"start": 17, "end": 22, "entity": "price", "value": "lo"}', + '{"start": 44, "end": 50, "entity": "cuisine", "value": "french"}', + '{"start": 74, "end": 80, "entity": "location", "value": "bombay"}', + '{"start": 4, "end": 7, "entity": "people", "value": "6"}', + '{"start": 18, "end": 28, "entity": "price", "value": "moderate"}', + ] + + assert story_evaluation.evaluation_store.serialise()[0] == serialised_actions assert not story_evaluation.evaluation_store.has_prediction_target_mismatch() assert len(story_evaluation.failed_stories) == 0 - assert num_stories == 2 + assert num_stories == 3 async def test_end_to_end_evaluation_script_unknown_entity(tmpdir, default_agent): diff --git a/tests/core/test_utils.py b/tests/core/test_utils.py index f9148cbaa701..8e9069d00d39 100644 --- a/tests/core/test_utils.py +++ b/tests/core/test_utils.py @@ -63,15 +63,6 @@ def test_cap_length_with_short_string(): assert utils.cap_length("my", 3) == "my" -def test_pad_list_to_size(): - assert utils.pad_list_to_size(["e1", "e2"], 4, "other") == [ - "e1", - "e2", - "other", - "other", - ] - - def test_read_lines(): lines = utils.read_lines( "data/test_stories/stories.md", max_line_limit=2, line_pattern=r"\*.*" From 380eb8b3a95f52b5a2675348bf6544e6b6e3ac00 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Thu, 18 Jul 2019 15:36:14 +0200 Subject: [PATCH 04/22] Update restaurantbot test --- tests/core/conftest.py | 5 +---- tests/core/test_evaluation.py | 4 ++-- tests/core/test_examples.py | 19 ++++--------------- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/tests/core/conftest.py b/tests/core/conftest.py index bf5d947783e3..d68e89e276de 100644 --- a/tests/core/conftest.py +++ b/tests/core/conftest.py @@ -7,10 +7,8 @@ import pytest import rasa.utils.io -from rasa.core import train from rasa.core.agent import Agent -from rasa.core.channels import channel -from rasa.core.channels.channel import CollectingOutputChannel, RestInput +from rasa.core.channels.channel import CollectingOutputChannel from rasa.core.domain import Domain from rasa.core.interpreter import RegexInterpreter from rasa.core.nlg import TemplatedNaturalLanguageGenerator @@ -24,7 +22,6 @@ from rasa.core.slots import Slot from rasa.core.tracker_store import InMemoryTrackerStore from rasa.core.trackers import DialogueStateTracker -from rasa.utils.io import zip_folder from rasa.train import train_async matplotlib.use("Agg") diff --git a/tests/core/test_evaluation.py b/tests/core/test_evaluation.py index 6e51851045a4..9d60ff5fed94 100644 --- a/tests/core/test_evaluation.py +++ b/tests/core/test_evaluation.py @@ -37,7 +37,7 @@ async def test_end_to_end_evaluation_script(tmpdir, restaurantbot): completed_trackers, restaurantbot, use_e2e=True ) - serialised_actions = [ + serialised_store = [ "utter_ask_howcanhelp", "action_listen", "utter_ask_howcanhelp", @@ -79,7 +79,7 @@ async def test_end_to_end_evaluation_script(tmpdir, restaurantbot): '{"start": 18, "end": 28, "entity": "price", "value": "moderate"}', ] - assert story_evaluation.evaluation_store.serialise()[0] == serialised_actions + assert story_evaluation.evaluation_store.serialise()[0] == serialised_store assert not story_evaluation.evaluation_store.has_prediction_target_mismatch() assert len(story_evaluation.failed_stories) == 0 assert num_stories == 3 diff --git a/tests/core/test_examples.py b/tests/core/test_examples.py index f450014a6f7f..d8714f573615 100644 --- a/tests/core/test_examples.py +++ b/tests/core/test_examples.py @@ -11,6 +11,7 @@ from rasa.core.train import train from rasa.core.utils import AvailableEndpoints from rasa.utils.endpoints import EndpointConfig, ClientResponseError +from tests.core.conftest import RESTAURANTBOT_MODEL_PATH @pytest.fixture(scope="session") @@ -89,20 +90,8 @@ async def test_formbot_example(): assert responses[0]["text"] == "chitchat" -async def test_restaurantbot_example(): - sys.path.append("examples/restaurantbot/") - from run import train_core, train_nlu, parse - - p = "examples/restaurantbot/" - stories = os.path.join("data", "test_stories", "stories_babi_small.md") - nlu_data = os.path.join(p, "data", "nlu.md") - await train_core( - os.path.join(p, "domain.yml"), os.path.join(p, "models"), "current", stories - ) - train_nlu( - os.path.join(p, "config.yml"), os.path.join(p, "models"), "current", nlu_data - ) - - responses = await parse("hello", os.path.join(p, "models", "current")) +async def test_restaurantbot_example(tmpdir): + restaurantbot = Agent.load_local_model(RESTAURANTBOT_MODEL_PATH) + responses = await restaurantbot.handle_text("Hello") assert responses[0]["text"] == "how can I help you?" From abf015e6fa388bb9f699a476683c2e69578562e3 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Mon, 22 Jul 2019 15:54:57 +0100 Subject: [PATCH 05/22] Add restaurantbot test back --- tests/core/test_examples.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/core/test_examples.py b/tests/core/test_examples.py index d8714f573615..f450014a6f7f 100644 --- a/tests/core/test_examples.py +++ b/tests/core/test_examples.py @@ -11,7 +11,6 @@ from rasa.core.train import train from rasa.core.utils import AvailableEndpoints from rasa.utils.endpoints import EndpointConfig, ClientResponseError -from tests.core.conftest import RESTAURANTBOT_MODEL_PATH @pytest.fixture(scope="session") @@ -90,8 +89,20 @@ async def test_formbot_example(): assert responses[0]["text"] == "chitchat" -async def test_restaurantbot_example(tmpdir): - restaurantbot = Agent.load_local_model(RESTAURANTBOT_MODEL_PATH) +async def test_restaurantbot_example(): + sys.path.append("examples/restaurantbot/") + from run import train_core, train_nlu, parse + + p = "examples/restaurantbot/" + stories = os.path.join("data", "test_stories", "stories_babi_small.md") + nlu_data = os.path.join(p, "data", "nlu.md") + await train_core( + os.path.join(p, "domain.yml"), os.path.join(p, "models"), "current", stories + ) + train_nlu( + os.path.join(p, "config.yml"), os.path.join(p, "models"), "current", nlu_data + ) + + responses = await parse("hello", os.path.join(p, "models", "current")) - responses = await restaurantbot.handle_text("Hello") assert responses[0]["text"] == "how can I help you?" From 06a85c09f111229d3e9b2b33daf52e18caa85759 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Mon, 22 Jul 2019 15:56:36 +0100 Subject: [PATCH 06/22] Add run.py back to restaurantbot code --- CHANGELOG.rst | 3 +- examples/restaurantbot/README.md | 6 ++ examples/restaurantbot/run.py | 114 +++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 examples/restaurantbot/run.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d64aaeb582c7..c4e7d7b4d94a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,8 +16,7 @@ Added Changed ------- -- restaurantbot model was replaced with a stacked model -- ``end_to_end_story.md`` was re-written in the restaurantbot domain + Removed ------- diff --git a/examples/restaurantbot/README.md b/examples/restaurantbot/README.md index 18824f615fff..da522c27ce03 100644 --- a/examples/restaurantbot/README.md +++ b/examples/restaurantbot/README.md @@ -1,5 +1,10 @@ # Restaurant Bot +This example includes a file called `run.py`, which contains an example +of how to use Rasa directly from your python code. + +## What’s inside this example? + This example contains some training data and the main files needed to build an assistant on your local machine. The `restaurantbot` consists of the following files: @@ -10,6 +15,7 @@ assistant on your local machine. The `restaurantbot` consists of the following f - **domain.yml** contains the domain of the assistant - **endpoints.yml** contains the webhook configuration for the custom action - **policy.py** contains a custom policy +- **run.py** contains code to train a Rasa model and use it to parse some text ## How to use this example? diff --git a/examples/restaurantbot/run.py b/examples/restaurantbot/run.py new file mode 100644 index 000000000000..2add336a84ca --- /dev/null +++ b/examples/restaurantbot/run.py @@ -0,0 +1,114 @@ +import argparse +import asyncio +import logging +from typing import Text + +import os +import rasa.utils.io +import rasa.train +from examples.restaurantbot.policy import RestaurantPolicy +from rasa.core.agent import Agent +from rasa.core.policies.memoization import MemoizationPolicy +from rasa.core.policies.mapping_policy import MappingPolicy + +logger = logging.getLogger(__name__) + + +async def parse(text: Text, model_path: Text): + agent = Agent.load(model_path) + + response = await agent.handle_text(text) + + logger.info("Text: '{}'".format(text)) + logger.info("Response:") + logger.info(response) + + return response + + +async def train_core( + domain_file: Text = "domain.yml", + model_directory: Text = "models", + model_name: Text = "current", + training_data_file: Text = "data/stories.md", +): + agent = Agent( + domain_file, + policies=[ + MemoizationPolicy(max_history=3), + MappingPolicy(), + RestaurantPolicy(batch_size=100, epochs=400, validation_split=0.2), + ], + ) + + training_data = await agent.load_data(training_data_file) + agent.train(training_data) + + # Attention: agent.persist stores the model and all meta data into a folder. + # The folder itself is not zipped. + model_path = os.path.join(model_directory, model_name, "core") + agent.persist(model_path) + + logger.info("Model trained. Stored in '{}'.".format(model_path)) + + return model_path + + +def train_nlu( + config_file="config.yml", + model_directory: Text = "models", + model_name: Text = "current", + training_data_file="data/nlu.md", +): + from rasa.nlu.training_data import load_data + from rasa.nlu import config + from rasa.nlu.model import Trainer + + training_data = load_data(training_data_file) + trainer = Trainer(config.load(config_file)) + trainer.train(training_data) + + # Attention: trainer.persist stores the model and all meta data into a folder. + # The folder itself is not zipped. + model_path = os.path.join(model_directory, model_name) + model_directory = trainer.persist(model_path, fixed_model_name="nlu") + + logger.info("Model trained. Stored in '{}'.".format(model_directory)) + + return model_directory + + +if __name__ == "__main__": + rasa.utils.io.configure_colored_logging(loglevel="INFO") + + parser = argparse.ArgumentParser(description="Restaurant Bot") + + subparser = parser.add_subparsers(dest="subparser_name") + train_parser = subparser.add_parser("train", help="train a core or nlu model") + parse_parser = subparser.add_parser("predict", help="predict next action") + + parse_parser.add_argument( + "--model", + default="models/current", + help="Path to the model directory which contains " + "sub-folders for core and nlu models.", + ) + parse_parser.add_argument("--text", default="hello", help="Text to parse.") + + train_parser.add_argument( + "model", + choices=["nlu", "core"], + help="Do you want to train a NLU or Core model?", + ) + args = parser.parse_args() + + loop = asyncio.get_event_loop() + + # decide what to do based on first parameter of the script + if args.subparser_name == "train": + if args.model == "nlu": + train_nlu() + elif args.model == "core": + loop.run_until_complete(train_core()) + elif args.subparser_name == "predict": + loop.run_until_complete(parse(args.text, args.model)) From 7ccb19d45fd941a3b76a24a1dbb7d2d4b5e24ec8 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Mon, 22 Jul 2019 17:14:43 +0100 Subject: [PATCH 07/22] Add pad_lists_to_size back in, add types, This reverts commit b733e0fe2b0db2838ee7903613a2968c85fc16a9. --- CHANGELOG.rst | 23 ++++++++++++++++++++--- rasa/core/test.py | 22 +++++++++------------- rasa/core/utils.py | 13 +++++++++++++ tests/core/test_utils.py | 10 ++++++++++ 4 files changed, 52 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c4e7d7b4d94a..64c2255c0f35 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,7 +6,7 @@ Rasa Change Log All notable changes to this project will be documented in this file. This project adheres to `Semantic Versioning`_ starting with version 1.0. -[Unreleased 1.1.7] - `master`_ +[Unreleased 1.1.8] - `master`_ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Added @@ -16,7 +16,9 @@ Added Changed ------- - +- ``Agent.update_model()`` and ``Agent.handle_message()`` now work without needing to set a domain + or a policy ensemble +- ``end_to_end_story.md`` was re-written in the restaurantbot domain Removed ------- @@ -24,9 +26,24 @@ Removed Fixed ----- -- validation no longer throws an error during interactive learning - messages with multiple entities are now handled properly with e2e evaluation +[1.1.7] - 2019-07-18 +^^^^^^^^^^^^^^^^^^^^ + +Added +----- +- added optional pymongo dependencies ``[tls, srv]`` to ``requirements.txt`` for better mongodb support +- ``case_sensitive`` option added to ``WhiteSpaceTokenizer`` with ``true`` as default. + +Fixed +----- +- validation no longer throws an error during interactive learning +- fixed wrong cleaning of ``use_entities`` in case it was a list and not ``True`` +- updated the server endpoint ``/model/parse`` to handle also messages with the intent prefix +- fixed bug where "No model found" message appeared after successfully running the bot +- debug logs now print to ``rasa_core.log`` when running ``rasa x -vv`` or ``rasa run -vv`` + [1.1.6] - 2019-07-12 ^^^^^^^^^^^^^^^^^^^^ diff --git a/rasa/core/test.py b/rasa/core/test.py index 1e53cae5d361..8fb3f9036bb7 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -1,17 +1,16 @@ import json import logging import os -import typing import warnings from collections import defaultdict, namedtuple from typing import Any, Dict, List, Optional, Text, Tuple from rasa.constants import RESULTS_FILE from rasa.core.events import ActionExecuted, UserUttered +from rasa.core.utils import pad_lists_to_size -if typing.TYPE_CHECKING: - from rasa.core.agent import Agent - from rasa.core.trackers import DialogueStateTracker +from rasa.core.agent import Agent +from rasa.core.trackers import DialogueStateTracker logger = logging.getLogger(__name__) @@ -76,7 +75,7 @@ def has_prediction_target_mismatch(self): or self.action_predictions != self.action_targets ) - def serialise(self): + def serialise(self) -> Tuple[List, List]: """Turn targets and predictions to lists of equal size for sklearn""" targets = ( @@ -91,10 +90,7 @@ def serialise(self): ) # sklearn does not cope with lists of unequal size, nor None values - padding = len(targets) - len(predictions) - predictions += ["None"] * padding - - return targets, predictions + return pad_lists_to_size(targets, predictions, padding_value="None") class WronglyPredictedAction(ActionExecuted): @@ -139,7 +135,7 @@ class WronglyClassifiedUserUtterance(UserUttered): def __init__(self, event: UserUttered, eval_store: EvaluationStore): - if eval_store.intent_predictions == list(): + if not eval_store.intent_predictions: self.predicted_intent = None else: self.predicted_intent = eval_store.intent_predictions[0] @@ -197,14 +193,14 @@ def _clean_entity_results(entity_results): def _collect_user_uttered_predictions( - event, partial_tracker, fail_on_prediction_errors -): + event: UserUttered, partial_tracker: DialogueStateTracker, fail_on_prediction_errors: bool +) -> EvaluationStore: user_uttered_eval_store = EvaluationStore() intent_gold = event.parse_data.get("true_intent") predicted_intent = event.parse_data.get("intent").get("name") - if predicted_intent == list(): + if not predicted_intent: predicted_intent = [None] user_uttered_eval_store.add_to_store( diff --git a/rasa/core/utils.py b/rasa/core/utils.py index 23dcf18080f2..d780d20675aa 100644 --- a/rasa/core/utils.py +++ b/rasa/core/utils.py @@ -376,6 +376,19 @@ def remove_none_values(obj: Dict[Text, Any]) -> Dict[Text, Any]: return {k: v for k, v in obj.items() if v is not None} +def pad_lists_to_size(list_x: List, list_y: List, padding_value: Optional[Any]=None) -> Tuple[List, List]: + """Compares list sizes and pads them to equal length.""" + + difference = len(list_x) - len(list_y) + + if difference > 0: + return list_x, list_y + [padding_value] * difference + elif difference < 0: + return list_x + [padding_value] * (-difference), list_y + else: + return list_x, list_y + + class AvailableEndpoints(object): """Collection of configured endpoints.""" diff --git a/tests/core/test_utils.py b/tests/core/test_utils.py index 8e9069d00d39..fa8fa0b99ddd 100644 --- a/tests/core/test_utils.py +++ b/tests/core/test_utils.py @@ -150,3 +150,13 @@ def test_file_in_path(file, parents): ) def test_file_not_in_path(file, parents): assert not rasa.utils.io.is_subdirectory(file, parents) + + +def test_pad_lists_to_size(): + list_x = [1, 2, 3] + list_y = ["a", "b"] + list_z = [None, None, None] + + assert utils.pad_lists_to_size(list_x, list_y) == (list_x, ["a", "b", None]) + assert utils.pad_lists_to_size(list_y, list_x, "c") == (["a", "b", "c"], list_x) + assert utils.pad_lists_to_size(list_z, list_x) == (list_z, list_x) From 5a23516080bc193e0a36394c23f4fd5f5941b160 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Mon, 22 Jul 2019 17:17:46 +0100 Subject: [PATCH 08/22] black --- CHANGELOG.rst | 1 - rasa/core/test.py | 4 +++- rasa/core/utils.py | 4 +++- tests/core/test_utils.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 64c2255c0f35..9c05c39fdd0a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,7 +11,6 @@ This project adheres to `Semantic Versioning`_ starting with version 1.0. Added ----- -- added optional pymongo dependencies ``[tls, srv]`` to ``requirements.txt`` for better mongodb support Changed diff --git a/rasa/core/test.py b/rasa/core/test.py index 8fb3f9036bb7..5830376415de 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -193,7 +193,9 @@ def _clean_entity_results(entity_results): def _collect_user_uttered_predictions( - event: UserUttered, partial_tracker: DialogueStateTracker, fail_on_prediction_errors: bool + event: UserUttered, + partial_tracker: DialogueStateTracker, + fail_on_prediction_errors: bool, ) -> EvaluationStore: user_uttered_eval_store = EvaluationStore() diff --git a/rasa/core/utils.py b/rasa/core/utils.py index d780d20675aa..1d8fe601e6bb 100644 --- a/rasa/core/utils.py +++ b/rasa/core/utils.py @@ -376,7 +376,9 @@ def remove_none_values(obj: Dict[Text, Any]) -> Dict[Text, Any]: return {k: v for k, v in obj.items() if v is not None} -def pad_lists_to_size(list_x: List, list_y: List, padding_value: Optional[Any]=None) -> Tuple[List, List]: +def pad_lists_to_size( + list_x: List, list_y: List, padding_value: Optional[Any] = None +) -> Tuple[List, List]: """Compares list sizes and pads them to equal length.""" difference = len(list_x) - len(list_y) diff --git a/tests/core/test_utils.py b/tests/core/test_utils.py index fa8fa0b99ddd..6b4cae267821 100644 --- a/tests/core/test_utils.py +++ b/tests/core/test_utils.py @@ -157,6 +157,6 @@ def test_pad_lists_to_size(): list_y = ["a", "b"] list_z = [None, None, None] - assert utils.pad_lists_to_size(list_x, list_y) == (list_x, ["a", "b", None]) + assert utils.pad_lists_to_size(list_x, list_y) == (list_x, ["a", "b", None]) assert utils.pad_lists_to_size(list_y, list_x, "c") == (["a", "b", "c"], list_x) assert utils.pad_lists_to_size(list_z, list_x) == (list_z, list_x) From 65cd7aaca84df04de6e9fe6a111db10d27e480a8 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Mon, 22 Jul 2019 17:34:58 +0100 Subject: [PATCH 09/22] Fix importing error --- rasa/core/test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rasa/core/test.py b/rasa/core/test.py index 5830376415de..95797628c3c4 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -2,16 +2,18 @@ import logging import os import warnings +import typing from collections import defaultdict, namedtuple from typing import Any, Dict, List, Optional, Text, Tuple from rasa.constants import RESULTS_FILE from rasa.core.events import ActionExecuted, UserUttered from rasa.core.utils import pad_lists_to_size - -from rasa.core.agent import Agent from rasa.core.trackers import DialogueStateTracker +if typing.TYPE_CHECKING: + from rasa.core.agent import Agent + logger = logging.getLogger(__name__) StoryEvalution = namedtuple( From 3e29b898e20d3ce0b4f36b1d3106e2bb21cfd984 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Mon, 22 Jul 2019 19:03:02 +0100 Subject: [PATCH 10/22] Fix test --- tests/core/conftest.py | 18 ++++++++++++++---- tests/core/test_evaluation.py | 2 ++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/core/conftest.py b/tests/core/conftest.py index d68e89e276de..5a78b6b80917 100644 --- a/tests/core/conftest.py +++ b/tests/core/conftest.py @@ -1,5 +1,4 @@ import asyncio -import logging import os from typing import Text @@ -40,7 +39,7 @@ MOODBOT_MODEL_PATH = "examples/moodbot/models/" -RESTAURANTBOT_MODEL_PATH = "examples/restaurantbot/models/" +RESTAURANTBOT_PATH = "examples/restaurantbot/" DEFAULT_ENDPOINTS_FILE = "data/test_endpoints/example_endpoints.yml" @@ -237,5 +236,16 @@ def trained_model(project) -> Text: @pytest.fixture -def restaurantbot() -> Agent: - return Agent.load_local_model(RESTAURANTBOT_MODEL_PATH) +async def restaurantbot(tmpdir_factory) -> Text: + model_path = tmpdir_factory.mktemp("model").strpath + restaurant_domain = os.path.join(RESTAURANTBOT_PATH, 'domain.yml') + restaurant_config = os.path.join(RESTAURANTBOT_PATH, 'config.yml') + restaurant_data = os.path.join(RESTAURANTBOT_PATH, 'data/') + + agent = await train_async( + restaurant_domain, + restaurant_config, + restaurant_data, + model_path + ) + return agent diff --git a/tests/core/test_evaluation.py b/tests/core/test_evaluation.py index 9d60ff5fed94..e9b7d8719e92 100644 --- a/tests/core/test_evaluation.py +++ b/tests/core/test_evaluation.py @@ -5,6 +5,7 @@ # we need this import to ignore the warning... # noinspection PyUnresolvedReferences from rasa.nlu.test import run_evaluation +from rasa.core.agent import Agent from tests.core.conftest import ( DEFAULT_STORIES_FILE, E2E_STORY_FILE_UNKNOWN_ENTITY, @@ -29,6 +30,7 @@ async def test_evaluation_image_creation(tmpdir, default_agent): async def test_end_to_end_evaluation_script(tmpdir, restaurantbot): + restaurantbot = Agent.load(restaurantbot) completed_trackers = await _generate_trackers( END_TO_END_STORY_FILE, restaurantbot, use_e2e=True ) From 2733e9537b59932da26028feec1d0c56164870e7 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Tue, 23 Jul 2019 18:46:28 +0100 Subject: [PATCH 11/22] Use dictionaries in serialisation instead of strings --- rasa/core/test.py | 10 ++-------- tests/core/conftest.py | 11 ++++------- tests/core/test_evaluation.py | 16 ++++++++-------- 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/rasa/core/test.py b/rasa/core/test.py index 95797628c3c4..b01c07356239 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -80,15 +80,9 @@ def has_prediction_target_mismatch(self): def serialise(self) -> Tuple[List, List]: """Turn targets and predictions to lists of equal size for sklearn""" - targets = ( - self.action_targets - + self.intent_targets - + [json.dumps(t) for t in self.entity_targets] - ) + targets = self.action_targets + self.intent_targets + self.entity_targets predictions = ( - self.action_predictions - + self.intent_predictions - + [json.dumps(p) for p in self.entity_predictions] + self.action_predictions + self.intent_predictions + self.entity_predictions ) # sklearn does not cope with lists of unequal size, nor None values diff --git a/tests/core/conftest.py b/tests/core/conftest.py index 5a78b6b80917..95e4ef93e7c6 100644 --- a/tests/core/conftest.py +++ b/tests/core/conftest.py @@ -238,14 +238,11 @@ def trained_model(project) -> Text: @pytest.fixture async def restaurantbot(tmpdir_factory) -> Text: model_path = tmpdir_factory.mktemp("model").strpath - restaurant_domain = os.path.join(RESTAURANTBOT_PATH, 'domain.yml') - restaurant_config = os.path.join(RESTAURANTBOT_PATH, 'config.yml') - restaurant_data = os.path.join(RESTAURANTBOT_PATH, 'data/') + restaurant_domain = os.path.join(RESTAURANTBOT_PATH, "domain.yml") + restaurant_config = os.path.join(RESTAURANTBOT_PATH, "config.yml") + restaurant_data = os.path.join(RESTAURANTBOT_PATH, "data/") agent = await train_async( - restaurant_domain, - restaurant_config, - restaurant_data, - model_path + restaurant_domain, restaurant_config, restaurant_data, model_path ) return agent diff --git a/tests/core/test_evaluation.py b/tests/core/test_evaluation.py index e9b7d8719e92..4d0ee7f3d69a 100644 --- a/tests/core/test_evaluation.py +++ b/tests/core/test_evaluation.py @@ -71,14 +71,14 @@ async def test_end_to_end_evaluation_script(tmpdir, restaurantbot): "inform", "inform", "deny", - '{"start": 17, "end": 27, "entity": "price", "value": "moderate"}', - '{"start": 53, "end": 57, "entity": "location", "value": "east"}', - '{"start": 0, "end": 6, "entity": "cuisine", "value": "french"}', - '{"start": 17, "end": 22, "entity": "price", "value": "lo"}', - '{"start": 44, "end": 50, "entity": "cuisine", "value": "french"}', - '{"start": 74, "end": 80, "entity": "location", "value": "bombay"}', - '{"start": 4, "end": 7, "entity": "people", "value": "6"}', - '{"start": 18, "end": 28, "entity": "price", "value": "moderate"}', + {"start": 17, "end": 27, "entity": "price", "value": "moderate"}, + {"start": 53, "end": 57, "entity": "location", "value": "east"}, + {"start": 0, "end": 6, "entity": "cuisine", "value": "french"}, + {"start": 17, "end": 22, "entity": "price", "value": "lo"}, + {"start": 44, "end": 50, "entity": "cuisine", "value": "french"}, + {"start": 74, "end": 80, "entity": "location", "value": "bombay"}, + {"start": 4, "end": 7, "entity": "people", "value": "6"}, + {"start": 18, "end": 28, "entity": "price", "value": "moderate"}, ] assert story_evaluation.evaluation_store.serialise()[0] == serialised_store From 990e8db340c6943bd681286c9830c879f09d23fc Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Tue, 23 Jul 2019 23:31:13 +0100 Subject: [PATCH 12/22] Use strings to enable comparisons --- rasa/core/test.py | 10 ++++++++-- tests/core/test_evaluation.py | 16 ++++++++-------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/rasa/core/test.py b/rasa/core/test.py index b01c07356239..95797628c3c4 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -80,9 +80,15 @@ def has_prediction_target_mismatch(self): def serialise(self) -> Tuple[List, List]: """Turn targets and predictions to lists of equal size for sklearn""" - targets = self.action_targets + self.intent_targets + self.entity_targets + targets = ( + self.action_targets + + self.intent_targets + + [json.dumps(t) for t in self.entity_targets] + ) predictions = ( - self.action_predictions + self.intent_predictions + self.entity_predictions + self.action_predictions + + self.intent_predictions + + [json.dumps(p) for p in self.entity_predictions] ) # sklearn does not cope with lists of unequal size, nor None values diff --git a/tests/core/test_evaluation.py b/tests/core/test_evaluation.py index 4d0ee7f3d69a..e9b7d8719e92 100644 --- a/tests/core/test_evaluation.py +++ b/tests/core/test_evaluation.py @@ -71,14 +71,14 @@ async def test_end_to_end_evaluation_script(tmpdir, restaurantbot): "inform", "inform", "deny", - {"start": 17, "end": 27, "entity": "price", "value": "moderate"}, - {"start": 53, "end": 57, "entity": "location", "value": "east"}, - {"start": 0, "end": 6, "entity": "cuisine", "value": "french"}, - {"start": 17, "end": 22, "entity": "price", "value": "lo"}, - {"start": 44, "end": 50, "entity": "cuisine", "value": "french"}, - {"start": 74, "end": 80, "entity": "location", "value": "bombay"}, - {"start": 4, "end": 7, "entity": "people", "value": "6"}, - {"start": 18, "end": 28, "entity": "price", "value": "moderate"}, + '{"start": 17, "end": 27, "entity": "price", "value": "moderate"}', + '{"start": 53, "end": 57, "entity": "location", "value": "east"}', + '{"start": 0, "end": 6, "entity": "cuisine", "value": "french"}', + '{"start": 17, "end": 22, "entity": "price", "value": "lo"}', + '{"start": 44, "end": 50, "entity": "cuisine", "value": "french"}', + '{"start": 74, "end": 80, "entity": "location", "value": "bombay"}', + '{"start": 4, "end": 7, "entity": "people", "value": "6"}', + '{"start": 18, "end": 28, "entity": "price", "value": "moderate"}', ] assert story_evaluation.evaluation_store.serialise()[0] == serialised_store From 787f2bf381b50c950f1d206cdf8066c8be21e910 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Wed, 24 Jul 2019 09:43:44 +0100 Subject: [PATCH 13/22] Use sorting --- rasa/core/test.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/rasa/core/test.py b/rasa/core/test.py index 95797628c3c4..67fa2d2ecf5e 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -51,7 +51,7 @@ def add_to_store( entity_targets: Optional[List[Dict[Text, Any]]] = None, ) -> None: """Add items or lists of items to the store""" - for k, v in locals().items(): + for k, v in sorted(locals().items()): if k != "self" and v: attr = getattr(self, k) if isinstance(v, list): @@ -77,7 +77,7 @@ def has_prediction_target_mismatch(self): or self.action_predictions != self.action_targets ) - def serialise(self) -> Tuple[List, List]: + def serialise(self) -> Tuple[List[Text], List[Text]]: """Turn targets and predictions to lists of equal size for sklearn""" targets = ( @@ -187,11 +187,20 @@ async def _generate_trackers(resource_name, agent, max_stories=None, use_e2e=Fal return g.generate() -def _clean_entity_results(entity_results): - return [ - {k: r[k] for k in ("start", "end", "entity", "value") if k in r} - for r in entity_results - ] +def _clean_entity_results( + entity_results: List[Dict[Text, Any]] +) -> List[Dict[Text, Any]]: + """Extract only the token variables from an entity dict.""" + cleaned_entities = [] + + for r in tuple(entity_results): + cleaned_entity = {} + for k in ("start", "end", "entity", "value"): + if k in set(r): + cleaned_entity.update({k: r[k]}) + cleaned_entities.append(cleaned_entity) + + return cleaned_entities def _collect_user_uttered_predictions( From f6bcd57f6699d1687b586941edf345f1e833db35 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Wed, 24 Jul 2019 15:35:51 +0100 Subject: [PATCH 14/22] Use markdown format for entities --- rasa/core/test.py | 23 +++++++++++++++-------- tests/core/test_evaluation.py | 16 ++++++++-------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/rasa/core/test.py b/rasa/core/test.py index 67fa2d2ecf5e..09ab45d21896 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -10,6 +10,7 @@ from rasa.core.events import ActionExecuted, UserUttered from rasa.core.utils import pad_lists_to_size from rasa.core.trackers import DialogueStateTracker +from rasa.nlu.training_data.formats.markdown import MarkdownWriter if typing.TYPE_CHECKING: from rasa.core.agent import Agent @@ -51,7 +52,7 @@ def add_to_store( entity_targets: Optional[List[Dict[Text, Any]]] = None, ) -> None: """Add items or lists of items to the store""" - for k, v in sorted(locals().items()): + for k, v in locals().items(): if k != "self" and v: attr = getattr(self, k) if isinstance(v, list): @@ -78,17 +79,23 @@ def has_prediction_target_mismatch(self): ) def serialise(self) -> Tuple[List[Text], List[Text]]: - """Turn targets and predictions to lists of equal size for sklearn""" + """Turn targets and predictions to lists of equal size for sklearn.""" targets = ( self.action_targets + self.intent_targets - + [json.dumps(t) for t in self.entity_targets] + + [ + MarkdownWriter._generate_entity_md(gold.get("text"), gold) + for gold in self.entity_targets + ] ) predictions = ( self.action_predictions + self.intent_predictions - + [json.dumps(p) for p in self.entity_predictions] + + [ + MarkdownWriter._generate_entity_md(predicted.get("text"), predicted) + for predicted in self.entity_predictions + ] ) # sklearn does not cope with lists of unequal size, nor None values @@ -188,13 +195,13 @@ async def _generate_trackers(resource_name, agent, max_stories=None, use_e2e=Fal def _clean_entity_results( - entity_results: List[Dict[Text, Any]] + text: Text, entity_results: List[Dict[Text, Any]] ) -> List[Dict[Text, Any]]: """Extract only the token variables from an entity dict.""" cleaned_entities = [] for r in tuple(entity_results): - cleaned_entity = {} + cleaned_entity = {"text": text} for k in ("start", "end", "entity", "value"): if k in set(r): cleaned_entity.update({k: r[k]}) @@ -225,8 +232,8 @@ def _collect_user_uttered_predictions( if entity_gold or predicted_entities: user_uttered_eval_store.add_to_store( - entity_targets=_clean_entity_results(entity_gold), - entity_predictions=_clean_entity_results(predicted_entities), + entity_targets=_clean_entity_results(event.text, entity_gold), + entity_predictions=_clean_entity_results(event.text, predicted_entities), ) if user_uttered_eval_store.has_prediction_target_mismatch(): diff --git a/tests/core/test_evaluation.py b/tests/core/test_evaluation.py index e9b7d8719e92..bc377623ea87 100644 --- a/tests/core/test_evaluation.py +++ b/tests/core/test_evaluation.py @@ -71,14 +71,14 @@ async def test_end_to_end_evaluation_script(tmpdir, restaurantbot): "inform", "inform", "deny", - '{"start": 17, "end": 27, "entity": "price", "value": "moderate"}', - '{"start": 53, "end": 57, "entity": "location", "value": "east"}', - '{"start": 0, "end": 6, "entity": "cuisine", "value": "french"}', - '{"start": 17, "end": 22, "entity": "price", "value": "lo"}', - '{"start": 44, "end": 50, "entity": "cuisine", "value": "french"}', - '{"start": 74, "end": 80, "entity": "location", "value": "bombay"}', - '{"start": 4, "end": 7, "entity": "people", "value": "6"}', - '{"start": 18, "end": 28, "entity": "price", "value": "moderate"}', + "[moderately](price:moderate)", + "[east](location)", + "[french](cuisine)", + "[cheap](price:lo)", + "[french](cuisine)", + "[bombay](location)", + "[six](people:6)", + "[moderately](price:moderate)", ] assert story_evaluation.evaluation_store.serialise()[0] == serialised_store From 04abca51d0b95d034b39612f0a2238d1de2802e4 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Thu, 25 Jul 2019 18:59:55 +0100 Subject: [PATCH 15/22] move type check imports --- rasa/core/test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rasa/core/test.py b/rasa/core/test.py index 09ab45d21896..435d38e6d847 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -1,4 +1,3 @@ -import json import logging import os import warnings @@ -7,13 +6,13 @@ from typing import Any, Dict, List, Optional, Text, Tuple from rasa.constants import RESULTS_FILE -from rasa.core.events import ActionExecuted, UserUttered from rasa.core.utils import pad_lists_to_size -from rasa.core.trackers import DialogueStateTracker from rasa.nlu.training_data.formats.markdown import MarkdownWriter if typing.TYPE_CHECKING: from rasa.core.agent import Agent + from rasa.core.trackers import DialogueStateTracker + from rasa.core.events import ActionExecuted, UserUttered logger = logging.getLogger(__name__) From 48d9d35f628c10f898a3f1a008b2fadc2b7910d0 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Thu, 25 Jul 2019 19:00:09 +0100 Subject: [PATCH 16/22] black --- tests/core/test_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/core/test_utils.py b/tests/core/test_utils.py index b4fc90227530..f37ca4d593ba 100644 --- a/tests/core/test_utils.py +++ b/tests/core/test_utils.py @@ -72,6 +72,7 @@ def test_read_lines(): assert len(lines) == 2 + def test_pad_lists_to_size(): list_x = [1, 2, 3] list_y = ["a", "b"] @@ -79,4 +80,4 @@ def test_pad_lists_to_size(): assert utils.pad_lists_to_size(list_x, list_y) == (list_x, ["a", "b", None]) assert utils.pad_lists_to_size(list_y, list_x, "c") == (["a", "b", "c"], list_x) - assert utils.pad_lists_to_size(list_z, list_x) == (list_z, list_x) \ No newline at end of file + assert utils.pad_lists_to_size(list_z, list_x) == (list_z, list_x) From 8e77037188a0ad2fb9d196131099bce979ecb539 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Fri, 26 Jul 2019 09:00:15 +0100 Subject: [PATCH 17/22] Add events to import --- rasa/core/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/core/test.py b/rasa/core/test.py index 435d38e6d847..57134b290de4 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -7,12 +7,12 @@ from rasa.constants import RESULTS_FILE from rasa.core.utils import pad_lists_to_size +from rasa.core.events import ActionExecuted, UserUttered from rasa.nlu.training_data.formats.markdown import MarkdownWriter if typing.TYPE_CHECKING: from rasa.core.agent import Agent from rasa.core.trackers import DialogueStateTracker - from rasa.core.events import ActionExecuted, UserUttered logger = logging.getLogger(__name__) From c1a8fb1106b7f0dcc79c20b1ad449aecdadfe21b Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Wed, 31 Jul 2019 18:09:10 +0200 Subject: [PATCH 18/22] change restaurantbot config --- examples/restaurantbot/config.yml | 2 +- examples/restaurantbot/run.py | 4 ++-- rasa/core/test.py | 2 +- tests/core/conftest.py | 2 ++ tests/core/test_restore.py | 4 +++- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/examples/restaurantbot/config.yml b/examples/restaurantbot/config.yml index 26f82e50914f..695005196169 100644 --- a/examples/restaurantbot/config.yml +++ b/examples/restaurantbot/config.yml @@ -11,7 +11,7 @@ pipeline: policies: - name: "examples.restaurantbot.policy.RestaurantPolicy" batch_size: 100 - epochs: 400 + epochs: 100 validation_split: 0.2 - name: MemoizationPolicy - name: MappingPolicy diff --git a/examples/restaurantbot/run.py b/examples/restaurantbot/run.py index 2add336a84ca..da4954d115f3 100644 --- a/examples/restaurantbot/run.py +++ b/examples/restaurantbot/run.py @@ -37,11 +37,11 @@ async def train_core( policies=[ MemoizationPolicy(max_history=3), MappingPolicy(), - RestaurantPolicy(batch_size=100, epochs=400, validation_split=0.2), + RestaurantPolicy(batch_size=100, epochs=100, validation_split=0.2), ], ) - training_data = await agent.load_data(training_data_file) + training_data = await agent.load_data(training_data_file, augmentation_factor=10) agent.train(training_data) # Attention: agent.persist stores the model and all meta data into a folder. diff --git a/rasa/core/test.py b/rasa/core/test.py index 57134b290de4..df36f971914b 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -9,10 +9,10 @@ from rasa.core.utils import pad_lists_to_size from rasa.core.events import ActionExecuted, UserUttered from rasa.nlu.training_data.formats.markdown import MarkdownWriter +from rasa.core.trackers import DialogueStateTracker if typing.TYPE_CHECKING: from rasa.core.agent import Agent - from rasa.core.trackers import DialogueStateTracker logger = logging.getLogger(__name__) diff --git a/tests/core/conftest.py b/tests/core/conftest.py index 95e4ef93e7c6..a09fa371ad1c 100644 --- a/tests/core/conftest.py +++ b/tests/core/conftest.py @@ -1,4 +1,5 @@ import asyncio +import logging import os from typing import Text @@ -6,6 +7,7 @@ import pytest import rasa.utils.io +from rasa.core import train from rasa.core.agent import Agent from rasa.core.channels.channel import CollectingOutputChannel from rasa.core.domain import Domain diff --git a/tests/core/test_restore.py b/tests/core/test_restore.py index 0e2bf15969a2..aedaf124f641 100644 --- a/tests/core/test_restore.py +++ b/tests/core/test_restore.py @@ -8,7 +8,7 @@ from rasa.model import get_model -@pytest.fixture(scope="module") +@pytest.fixture(scope="session") def loop(): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -28,6 +28,8 @@ async def test_restoring_tracker(trained_moodbot_path, recwarn): # makes sure there are no warnings. warnings are raised, if the models # predictions differ from the tracker when the dumped tracker is replayed + + print([e.message for e in recwarn if e._category_name == "UserWarning"]) assert [e for e in recwarn if e._category_name == "UserWarning"] == [] assert len(tracker.events) == 7 From 6e4616c12e214c86e4c7d0c440ca145d2cdfb8e0 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Thu, 1 Aug 2019 10:55:05 +0200 Subject: [PATCH 19/22] Update test_restore This reverts commit c1a8fb1106b7f0dcc79c20b1ad449aecdadfe21b. --- tests/core/test_restore.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/core/test_restore.py b/tests/core/test_restore.py index aedaf124f641..0e2bf15969a2 100644 --- a/tests/core/test_restore.py +++ b/tests/core/test_restore.py @@ -8,7 +8,7 @@ from rasa.model import get_model -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def loop(): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -28,8 +28,6 @@ async def test_restoring_tracker(trained_moodbot_path, recwarn): # makes sure there are no warnings. warnings are raised, if the models # predictions differ from the tracker when the dumped tracker is replayed - - print([e.message for e in recwarn if e._category_name == "UserWarning"]) assert [e for e in recwarn if e._category_name == "UserWarning"] == [] assert len(tracker.events) == 7 From a14b2ae5906a03c9b5c0d57e6a7a9d60a7a7ec7c Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Mon, 5 Aug 2019 19:45:06 +0200 Subject: [PATCH 20/22] review commentts --- CHANGELOG.rst | 5 ++--- rasa/core/test.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 954fd84f1bf2..b1a232e90848 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,7 +16,7 @@ Added Changed ------- - +- ``data/test_evaluations/end_to_end_story.md`` was re-written in the restaurantbot domain Removed ------- @@ -24,6 +24,7 @@ Removed Fixed ----- +- messages with multiple entities are now handled properly with e2e evaluation [1.2.0] - 2019-08-01 @@ -40,7 +41,6 @@ Changed ------- - ``Agent.update_model()`` and ``Agent.handle_message()`` now work without needing to set a domain or a policy ensemble -- ``end_to_end_story.md`` was re-written in the restaurantbot domain - Update pytype to ``2019.7.11`` - new event broker class: ``SQLProducer``. This event broker is now used when running locally with Rasa X @@ -72,7 +72,6 @@ Changed Fixed ----- -- messages with multiple entities are now handled properly with e2e evaluation - interactive learning bug where reverted user utterances were dumped to training data - added timeout to terminal input channel to avoid freezing input in case of server errors diff --git a/rasa/core/test.py b/rasa/core/test.py index 0b76f516c0c3..77742ad48ef2 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -203,7 +203,7 @@ def _clean_entity_results( cleaned_entity = {"text": text} for k in ("start", "end", "entity", "value"): if k in set(r): - cleaned_entity.update({k: r[k]}) + cleaned_entity[k] = r[k] cleaned_entities.append(cleaned_entity) return cleaned_entities From 66f6746bfc534daae2affc52410d88b2c5b697d4 Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Wed, 7 Aug 2019 11:46:45 +0200 Subject: [PATCH 21/22] update version --- rasa/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/version.py b/rasa/version.py index a955fdae12bd..bc86c944fe22 100644 --- a/rasa/version.py +++ b/rasa/version.py @@ -1 +1 @@ -__version__ = "1.2.1" +__version__ = "1.2.2" From 614b3c86982d35862aa8ee4208ec0c2400f629fe Mon Sep 17 00:00:00 2001 From: Tom Metcalfe Date: Wed, 7 Aug 2019 15:01:09 +0200 Subject: [PATCH 22/22] resolve type check --- rasa/core/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/core/test.py b/rasa/core/test.py index 77742ad48ef2..254190ab0e78 100644 --- a/rasa/core/test.py +++ b/rasa/core/test.py @@ -217,7 +217,7 @@ def _collect_user_uttered_predictions( user_uttered_eval_store = EvaluationStore() intent_gold = event.parse_data.get("true_intent") - predicted_intent = event.parse_data.get("intent").get("name") + predicted_intent = event.parse_data.get("intent", {}).get("name") if not predicted_intent: predicted_intent = [None]