Merge e1b9378 into 7197472

CHANGELOG.rst

@@ -16,6 +16,10 @@ Added
 - A support for session persistence mechanism in the ``SocketIOInput``
   compatible with the example SocketIO WebChat + short explanation on
   how session persistence should be implemented in a frontend
+- `TwoStageFallbackPolicy` which asks the user for their affirmation if the NLU
+  confidence is low for an intent, for rephrasing the intent if they deny the
+  suggested intent, and does finally an ultimate fallback if it does not get
+  the intent right
 
 Removed
 -------

docs/docker_walkthrough.rst

@@ -1,7 +1,9 @@
+:desc: Building Rasa with Docker
+
 .. _docker_walkthrough:
 
-Development with Docker
-=======================
+Building Rasa with Docker
+=========================
 
 This walkthrough provides a tutorial on how to set up Rasa Core, Rasa NLU,
 and an Action Server with Docker containers.

docs/policies.rst

@@ -5,6 +5,7 @@
 Training and Policies
 =====================
 
+.. contents::
 
 Training
 --------
@@ -218,8 +219,8 @@ or initialize ``KerasPolicy`` with pre-defined ``keras model``.
 
 .. _embedding_policy:
 
-Embedding policy
-----------------
+Embedding Policy
+^^^^^^^^^^^^^^^^
 
 The Recurrent Embedding Dialogue Policy (REDP)
 described in our paper: `<https://arxiv.org/abs/1811.11707>`_
@@ -393,5 +394,77 @@ It is recommended to use
           ``mu_neg = mu_pos`` and ``use_max_sim_neg = False``. See
           `starspace paper <https://arxiv.org/abs/1709.03856>`_ for details.
 
+Two-stage Fallback Policy
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This policy handles low NLU confidence in multiple stages.
+
+- If a NLU prediction has a low confidence score, the user is asked to affirm
+  the classification of the intent.
+
+    - If they affirm, the story continues as if the intent was classified
+      with high confidence from the beginning.
+    - If they deny, the user is asked to rephrase their message.
+
+- Rephrasing
+
+    - If the classification of the rephrased intent was confident, the story
+      continues as if the user had this intent from the beginning.
+    - If the rephrased intent was not classified with high confidence, the user
+      is asked to affirm the classified intent.
+
+- Second affirmation
+
+    - If the user affirms the intent, the story continues as if the user had
+      this intent from the beginning.
+    - If the user denies, an ultimate fallback action is triggered
+      (e.g. a handoff to a human).
+
+Configuration
+"""""""""""""
+
+To use this policy, include the following in your policy configuration.
+Note that you cannot use this together with the default fallback policy.
+
+.. code-block:: yaml
+
+    policies:
+      - name: TwoStageFallbackPolicy
+        nlu_threshold: 0.3
+        core_threshold: 0.3
+        fallback_action_name: "action_default_fallback"
+
++-------------------------------+---------------------------------------------+
+| ``nlu_threshold``             | Min confidence needed to accept an NLU      |
+|                               | prediction                                  |
++-------------------------------+---------------------------------------------+
+| ``core_threshold``            | Min confidence needed to accept an action   |
+|                               | prediction from Rasa Core                   |
++-------------------------------+---------------------------------------------+
+| ``fallback_action_name``      | Name of the action to be called if the      |
+|                               | confidence of intent / action prediction    |
+|                               | is below the threshold                      |
++-------------------------------+---------------------------------------------+
+
+.. note::
+
+    It is required to have the two intents ``affirm`` and ``deny`` in the
+    domain of the bot, to determine whether the user affirms or
+    denies a suggestion.
+
+Default Actions for Affirmation and Rephrasing
+""""""""""""""""""""""""""""""""""""""""""""""
+
+Rasa Core provides the default implementations
+``action_default_ask_affirmation`` and ``action_default_ask_rephrase``
+which are triggered when the bot should ask the user to affirm
+or to rephrase their intent.
+The default implementation of ``action_default_ask_rephrase`` action utters
+the response template ``utter_ask_rephrase``.
+The implementation of both actions can be overwritten with :ref:`customactions`.
+
+
+
+
 
 .. include:: feedback.inc

rasa_core/actions/action.py

@@ -1,20 +1,21 @@
 import logging
-
-import requests
 import typing
 from typing import List, Text, Optional, Dict, Any
 
+import requests
+
 from rasa_core import events
 from rasa_core.constants import (
     DOCS_BASE_URL,
     DEFAULT_REQUEST_TIMEOUT,
-    REQUESTED_SLOT)
+    REQUESTED_SLOT, FALLBACK_SCORE, USER_INTENT_AFFIRM, USER_INTENT_DENY)
+from rasa_core.events import (UserUtteranceReverted, UserUttered,
+                              ActionExecuted, Event)
 from rasa_core.utils import EndpointConfig
 
 if typing.TYPE_CHECKING:
     from rasa_core.trackers import DialogueStateTracker
     from rasa_core.dispatcher import Dispatcher
-    from rasa_core.events import Event
     from rasa_core.domain import Domain
 
 logger = logging.getLogger(__name__)
@@ -27,16 +28,22 @@
 
 ACTION_DEACTIVATE_FORM_NAME = "action_deactivate_form"
 
+ACTION_REVERT_FALLBACK_EVENTS_NAME = 'action_revert_fallback_events'
+
+ACTION_DEFAULT_ASK_AFFIRMATION_NAME = 'action_default_ask_affirmation'
+
+ACTION_DEFAULT_ASK_REPHRASE_NAME = 'action_default_ask_rephrase'
 
-def default_actions():
-    # type: () -> List[Action]
+
+def default_actions() -> List['Action']:
     """List default actions."""
     return [ActionListen(), ActionRestart(),
-            ActionDefaultFallback(), ActionDeactivateForm()]
+            ActionDefaultFallback(), ActionDeactivateForm(),
+            ActionRevertFallbackEvents(), ActionDefaultAskAffirmation(),
+            ActionDefaultAskRephrase()]
 
 
-def default_action_names():
-    # type: () -> List[Text]
+def default_action_names() -> List[Text]:
     """List default action names."""
     return [a.name() for a in default_actions()]
 
@@ -94,8 +101,7 @@ def actions_from_names(action_names: List[Text],
 class Action(object):
     """Next action to be taken in response to a dialogue state."""
 
-    def name(self):
-        # type: () -> Text
+    def name(self) -> Text:
         """Unique identifier of this simple action."""
 
         raise NotImplementedError
@@ -121,7 +127,7 @@ def run(self, dispatcher: 'Dispatcher', tracker: 'DialogueStateTracker',
 
         raise NotImplementedError
 
-    def __str__(self):
+    def __str__(self) -> Text:
         return "Action('{}')".format(self.name())
 
 
@@ -141,10 +147,10 @@ def run(self, dispatcher, tracker, domain):
                                   tracker)
         return []
 
-    def name(self):
+    def name(self) -> Text:
         return self._name
 
-    def __str__(self):
+    def __str__(self) -> Text:
         return "UtterAction('{}')".format(self.name())
 
 
@@ -154,7 +160,7 @@ class ActionListen(Action):
     The bot should stop taking further actions and wait for the user to say
     something."""
 
-    def name(self):
+    def name(self) -> Text:
         return ACTION_LISTEN_NAME
 
     def run(self, dispatcher, tracker, domain):
@@ -166,7 +172,7 @@ class ActionRestart(Action):
 
     Utters the restart template if available."""
 
-    def name(self):
+    def name(self) -> Text:
         return ACTION_RESTART_NAME
 
     def run(self, dispatcher, tracker, domain):
@@ -182,7 +188,7 @@ class ActionDefaultFallback(Action):
     """Executes the fallback action and goes back to the previous state
     of the dialogue"""
 
-    def name(self):
+    def name(self) -> Text:
         return ACTION_DEFAULT_FALLBACK_NAME
 
     def run(self, dispatcher, tracker, domain):
@@ -197,7 +203,7 @@ def run(self, dispatcher, tracker, domain):
 class ActionDeactivateForm(Action):
     """Deactivates a form"""
 
-    def name(self):
+    def name(self) -> Text:
         return ACTION_DEACTIVATE_FORM_NAME
 
     def run(self, dispatcher, tracker, domain):
@@ -361,7 +367,7 @@ def run(self, dispatcher, tracker, domain):
 
         return evts
 
-    def name(self):
+    def name(self) -> Text:
         return self._name
 
 
@@ -377,3 +383,124 @@ def __init__(self, action_name, message=None):
 
     def __str__(self):
         return self.message
+
+
+class ActionRevertFallbackEvents(Action):
+    """Reverts events which were done during the `TwoStageFallbackPolicy`.
+
+       This reverts user messages and bot utterances done during a fallback
+       of the `TwoStageFallbackPolicy`. By doing so it is not necessary to
+       write custom stories for the different paths, but only of the happy
+       path.
+    """
+
+    def name(self) -> Text:
+        return ACTION_REVERT_FALLBACK_EVENTS_NAME
+
+    def run(self, dispatcher: 'Dispatcher', tracker: 'DialogueStateTracker',
+            domain: 'Domain') -> List[Event]:
+        from rasa_core.policies.two_stage_fallback import (has_user_rephrased,
+                                                           has_user_affirmed)
+
+        last_user_event = tracker.latest_message.intent.get('name')
+        revert_events = []
+
+        # User affirmed
+        if has_user_affirmed(last_user_event, tracker):
+            revert_events = _revert_affirmation_events(tracker)
+        # User rephrased
+        elif has_user_rephrased(tracker):
+            revert_events = _revert_successful_affirmation(tracker)
+        # User rephrased instead of affirming
+        elif tracker.last_executed_action_has(
+                ACTION_DEFAULT_ASK_AFFIRMATION_NAME):
+            revert_events = _revert_early_rephrasing(tracker)
+
+        return revert_events
+
+
+def _revert_affirmation_events(tracker: 'DialogueStateTracker') -> List[Event]:
+    import copy
+    revert_events = _revert_single_affirmation_events()
+
+    last_user_event = tracker.get_last_event_for(UserUttered, skip=1)
+    last_user_event = copy.deepcopy(last_user_event)
+    last_user_event.parse_data['intent']['confidence'] = FALLBACK_SCORE
+
+    # User affirms the rephrased intent
+    rephrased_intent = tracker.last_executed_action_has(
+        name=ACTION_DEFAULT_ASK_REPHRASE_NAME,
+        skip=1)
+    if rephrased_intent:
+        revert_events += _revert_rephrasing_events()
+
+    return revert_events + [last_user_event]
+
+
+def _revert_single_affirmation_events() -> List[Event]:
+    return [UserUtteranceReverted(),  # revert affirmation and request
+            # revert original intent (has to be re-added later)
+            UserUtteranceReverted(),
+            # add action listen intent
+            ActionExecuted(action_name=ACTION_LISTEN_NAME)]
+
+
+def _revert_successful_affirmation(tracker) -> List[Event]:
+    last_user_event = tracker.get_last_event_for(UserUttered)
+    return _revert_rephrasing_events() + [last_user_event]
+
+
+def _revert_early_rephrasing(tracker: 'DialogueStateTracker') -> List[Event]:
+    last_user_event = tracker.get_last_event_for(UserUttered)
+    return _revert_single_affirmation_events() + [last_user_event]
+
+
+def _revert_rephrasing_events() -> List[Event]:
+    return [UserUtteranceReverted(),  # remove rephrasing
+            # remove feedback and rephrase request
+            UserUtteranceReverted(),
+            # remove affirmation request and false intent
+            UserUtteranceReverted(),
+            # replace action with action listen
+            ActionExecuted(action_name=ACTION_LISTEN_NAME)]
+
+
+class ActionDefaultAskAffirmation(Action):
+    """Default implementation which asks the user to affirm his intent.
+
+       It is suggested to overwrite this default action with a custom action
+       to have more meaningful prompts for the affirmations. E.g. have a
+       description of the intent instead of its identifier name.
+    """
+
+    def name(self) -> Text:
+        return ACTION_DEFAULT_ASK_AFFIRMATION_NAME
+
+    def run(self, dispatcher: 'Dispatcher', tracker: 'DialogueStateTracker',
+            domain: 'Domain') -> List[Event]:
+        intent_to_affirm = tracker.latest_message.intent.get('name')
+        affirmation_message = "Did you mean '{}'?".format(intent_to_affirm)
+
+        dispatcher.utter_button_message(text=affirmation_message,
+                                        buttons=[{'title': 'Yes',
+                                                  'payload': '/{}'.format(
+                                                      USER_INTENT_AFFIRM)},
+                                                 {'title': 'No',
+                                                  'payload': '/{}'.format(
+                                                      USER_INTENT_DENY)}])
+
+        return []
+
+
+class ActionDefaultAskRephrase(Action):
+    """Default implementation which asks the user to rephrase his intent."""
+
+    def name(self) -> Text:
+        return ACTION_DEFAULT_ASK_REPHRASE_NAME
+
+    def run(self, dispatcher: 'Dispatcher', tracker: 'DialogueStateTracker',
+            domain: 'Domain') -> List[Event]:
+        dispatcher.utter_template("utter_ask_rephrase", tracker,
+                                  silent_fail=True)
+
+        return []

rasa_core/constants.py

@@ -26,3 +26,7 @@
 INTENT_MESSAGE_PREFIX = "/"
 
 USER_INTENT_RESTART = INTENT_MESSAGE_PREFIX + "restart"
+
+USER_INTENT_AFFIRM = 'affirm'
+
+USER_INTENT_DENY = 'deny'

rasa_core/policies/__init__.py

@@ -6,3 +6,4 @@
     MemoizationPolicy, AugmentedMemoizationPolicy)
 from rasa_core.policies.sklearn_policy import SklearnPolicy
 from rasa_core.policies.form_policy import FormPolicy
+from rasa_core.policies.two_stage_fallback import TwoStageFallbackPolicy