Merge branch 'master' into pub_2019_CoreTransformer_core

.travis.yml

@@ -114,4 +114,4 @@ jobs:
       distributions: "sdist bdist_wheel"
       password:
         secure: "MeL1Ve97eBY+VbNWuQNuLzkPs0TPc+Zh8OfZkhw69ez5imsiWpvp0LrUOLVW3CcC0vNTANEBOVX/n1kHxfcqkf/cChNqAkZ6zTMmvR9zHDwQxXVGZ3jEQSQM+fHdQpjwtH7BwojyxaCIC/5iza7DFMcca/Q6Xr+atdTd0V8Q7Nc5jFHEQf3/4oIIm6YeCUiHcEu981LRdS04+jvuFUN0Ejy+KLukGVyIWyYDjjGjs880Mj4J1mgmCihvVkJ1ujB65rYBdTjls3JpP3eTk63+xH8aHilIuvqB8TDYih8ovE/Vv6YwLI+u2HoEHAtBD4Ez3r71Ju6JKJM7DhWb5aurN4M7K6DC8AvpUl+PsJbNP4ZeW2jXMH6lT6qXKVaSw7lhZ0XY3wunyVcAbArX4RS0B9pb1nHBYUBWZjxXtr8lhkpGFu7H43hw63Y19qb8z4+1cGnijgz1mqXSAssuc+3r0W0cSr+OsCjmOs7cwT6HMQvPEKxLohwBOS/I3EbuKQOYMjFN5BWP5JXbsG45awV9tquxEW8zxjMetR+AOcYoyrDeiR8sAnj1/F99DE0bL1KyW/G5VNu2Xi/c+0M3KvP3+F8XTCuUY/5zTvqh1Qz1jcdiwsiAhO4eBQzQnjeFlxdiVeue2kmD5qsh+VLKKuKLfyVoaV7b1kBlAtBDu7+hDpA="
-    after_deploy: bash scripts/ping_slack_about_package_release.sh
+    after_deploy: ./scripts/ping_slack_about_package_release.sh

CHANGELOG.rst

@@ -12,17 +12,31 @@ This project adheres to `Semantic Versioning`_ starting with version 1.0.
 
 Added
 -----
+- added ``DynamoTrackerStore`` to support persistence of agents running on AWS
+- added docstrings for ``TrackerStore`` classes
+- added buttons and images to mattermost.
 
 Changed
 -------
+- Unknown sections in markdown format (NLU data) are not ignored anymore, but instead an error is raised.
 
 Removed
 -------
 
 Fixed
 -----
-- fix missing ``tkinter`` dependency for running tests on Ubuntu
+- fixed missing ``tkinter`` dependency for running tests on Ubuntu
+- fixed issue with ``conversation`` JSON serialization
 
+[1.3.4] - 2019-09-14
+^^^^^^^^^^^^^^^^^^^^
+
+Added
+-----
+- Added the ability to set the ``backlog`` parameter in Sanics ``run()`` method using
+  the ``SANIC_BACKLOG`` environment variable. This parameter sets the
+  number of unaccepted connections the server allows before refusing new
+  connections. A default value of 100 is used if the variable is not set.
 
 [1.3.3] - 2019-09-13
 ^^^^^^^^^^^^^^^^^^^^
@@ -35,6 +49,7 @@ Fixed
 - Default one-hot representation for label featurization inside ``EmbeddingIntentClassifier`` if label features don't exist.
 - Policy ensemble no longer incorrectly wrings "missing mapping policy" when
   mapping policy is present.
+- "test" from ``utter_custom_json`` now correctly saved to tracker when using telegram channel
 
 Removed
 -------

data/test/markdown_single_sections/empty_section.md

@@ -0,0 +1,8 @@
+## regex:greet
+- hey[^\s]*
+
+## synonym:animal
+
+## lookup:chinese
+- Chines
+- Chinese

data/test/markdown_single_sections/not_existing_section.md

@@ -0,0 +1,8 @@
+## synonym:animal
+
+## entity:human
+- person
+
+## lookup:chinese
+- Chines
+- Chinese

docs/conf.py

@@ -340,9 +340,9 @@
     re.compile(r"^[2-9]+\.\d+\.\d+$"),
     re.compile(r"^1\.[456789]+\.\d+$"),
     re.compile(r"^1\.3\.\d+$"),
-    "1.2.7",
-    "1.1.8",
-    "1.0.9",
+    re.compile("^1\.2\.9$"),
+    re.compile("^1\.1\.8$"),
+    re.compile("^1\.0\.9$"),
 )
 scv_greatest_tag = True
 

docs/user-guide/connectors/mattermost.rst

@@ -51,7 +51,8 @@ you need to supply a ``credentials.yml`` with the following content:
      team: "community"
      user: "user@user.com"
      pw: "password"
+     webhook_url: "https://server.example.com/webhooks/mattermost/webhook"
 
 The endpoint for receiving Mattermost channel messages
-is ``/webhooks/mattermost/webhook``. This is the url you should
-add in the Mattermost outgoing webhook.
+is ``/webhooks/mattermost/webhook``, the same as ``webhook_url`` here. You should
+add this url also in the Mattermost outgoing webhook.

examples/nlg_server/nlg_server.py

@@ -1,5 +1,6 @@
 import argparse
 import logging
+import os
 
 from sanic import Sanic, response
 
@@ -70,7 +71,12 @@ async def nlg(request):
 
         return response.json(bot_response)
 
-    app.run(host="0.0.0.0", port=port, workers=workers)
+    app.run(
+        host="0.0.0.0",
+        port=port,
+        workers=workers,
+        backlog=int(os.environ.get("SANIC_BACKLOG", "100")),
+    )
 
 
 if __name__ == "__main__":

rasa/cli/initial_project/credentials.yml

@@ -21,5 +21,12 @@ rest:
 #  bot_message_evt: <event name for but messages>
 #  session_persistence: <true/false>
 
+#mattermost:
+#  url: "https://<mattermost instance>/api/v4"
+#  team: "<mattermost team>"
+#  user: "<bot username>"
+#  pw: "<bot token>"
+#  webhook_url: "<callback URL>"
+
 rasa:
   url: "http://localhost:5002/api"

rasa/core/agent.py

@@ -711,7 +711,11 @@ def handle_channels(
 
         update_sanic_log_level()
 
-        app.run(host="0.0.0.0", port=http_port)
+        app.run(
+            host="0.0.0.0",
+            port=http_port,
+            backlog=int(os.environ.get("SANIC_BACKLOG", "100")),
+        )
 
         # this might seem unnecessary (as run does not return until the server
         # is killed) - but we use it for tests where we mock `.run` to directly

rasa/core/channels/mattermost.py

@@ -2,7 +2,7 @@
 from mattermostwrapper import MattermostAPI
 from sanic import Blueprint, response
 from sanic.request import Request
-from typing import Text, Dict, Any
+from typing import Text, Dict, Any, List, Callable, Awaitable
 
 from rasa.core.channels.channel import UserMessage, OutputChannel, InputChannel
 
@@ -16,12 +16,28 @@ class MattermostBot(MattermostAPI, OutputChannel):
     def name(cls):
         return "mattermost"
 
-    def __init__(self, url, team, user, pw, bot_channel):
+    @classmethod
+    def from_credentials(cls, credentials):
+        if not credentials:
+            cls.raise_missing_credentials_exception()
+
+        return cls(credentials.get("webhook_url"))
+
+    def __init__(
+        self,
+        url: Text,
+        team: Text,
+        user: Text,
+        pw: Text,
+        bot_channel: Text,
+        webhook_url: Text,
+    ):
         self.url = url
         self.team = team
         self.user = user
         self.pw = pw
         self.bot_channel = bot_channel
+        self.webhook_url = webhook_url
 
         super(MattermostBot, self).__init__(url, team)
         super(MattermostBot, self).login(user, pw)
@@ -39,6 +55,53 @@ async def send_custom_json(
         json_message.setdefault("message", "")
         self.post("/posts", json_message)
 
+    async def send_image_url(
+        self, recipient_id: Text, image: Text, **kwargs: Any
+    ) -> None:
+        """Sends an image."""
+        image_url = image
+
+        props = {"attachments": []}
+        props["attachments"].append({"image_url": image_url})
+
+        json_message = {}
+        json_message.setdefault("channel_id", self.bot_channel)
+        json_message.setdefault("props", props)
+
+        self.post("/posts", json_message)
+
+    async def send_text_with_buttons(
+        self,
+        recipient_id: Text,
+        text: Text,
+        buttons: List[Dict[Text, Any]],
+        **kwargs: Any
+    ) -> None:
+        """Sends buttons to the output."""
+
+        # buttons are a list of objects: [(option_name, payload)]
+        # See https://docs.mattermost.com/developer/interactive-messages.html#message-buttons
+        button_block = {"actions": []}
+        for button in buttons:
+            button_block["actions"].append(
+                {
+                    "name": button["title"],
+                    "integration": {
+                        "url": self.webhook_url,
+                        "context": {"action": button["payload"]},
+                    },
+                }
+            )
+        props = {"attachments": []}
+        props["attachments"].append(button_block)
+
+        json_message = {}
+        json_message.setdefault("channel_id", self.bot_channel)
+        json_message.setdefault("message", text)
+        json_message.setdefault("props", props)
+
+        self.post("/posts", json_message)
+
 
 class MattermostInput(InputChannel):
     """Mattermost input channel implemenation."""
@@ -57,9 +120,12 @@ def from_credentials(cls, credentials):
             credentials.get("team"),
             credentials.get("user"),
             credentials.get("pw"),
+            credentials.get("webhook_url"),
         )
 
-    def __init__(self, url: Text, team: Text, user: Text, pw: Text) -> None:
+    def __init__(
+        self, url: Text, team: Text, user: Text, pw: Text, webhook_url: Text
+    ) -> None:
         """Create a Mattermost input channel.
         Needs a couple of settings to properly authenticate and validate
         messages.
@@ -70,13 +136,75 @@ def __init__(self, url: Text, team: Text, user: Text, pw: Text) -> None:
             team: Your mattermost team name
             user: Your mattermost userid that will post messages
             pw: Your mattermost password for your user
+            webhook_url: The mattermost callback url as specified
+                in the outgoing webhooks in mattermost example
+                https://mysite.example.com/webhooks/mattermost/webhook
         """
         self.url = url
         self.team = team
         self.user = user
         self.pw = pw
+        self.webhook_url = webhook_url
 
-    def blueprint(self, on_new_message):
+    async def message_with_trigger_word(
+        self,
+        on_new_message: Callable[[UserMessage], Awaitable[None]],
+        output: Dict[Text, Any],
+    ) -> None:
+        # splitting to get rid of the @botmention
+        # trigger we are using for this
+        text = output["text"].split(" ", 1)
+        text = text[1]
+
+        sender_id = output["user_id"]
+        self.bot_channel = output["channel_id"]
+
+        try:
+            out_channel = MattermostBot(
+                self.url,
+                self.team,
+                self.user,
+                self.pw,
+                self.bot_channel,
+                self.webhook_url,
+            )
+            user_msg = UserMessage(
+                text, out_channel, sender_id, input_channel=self.name()
+            )
+            await on_new_message(user_msg)
+        except Exception as e:
+            logger.error("Exception when trying to handle message.{0}".format(e))
+            logger.debug(e, exc_info=True)
+
+    async def action_from_button(
+        self,
+        on_new_message: Callable[[UserMessage], Awaitable[None]],
+        output: Dict[Text, Any],
+    ) -> None:
+        # get the action, the buttons triggers
+        action = output["context"]["action"]
+
+        sender_id = output["user_id"]
+        self.bot_channel = output["channel_id"]
+
+        try:
+            out_channel = MattermostBot(
+                self.url,
+                self.team,
+                self.user,
+                self.pw,
+                self.bot_channel,
+                self.webhook_url,
+            )
+            context_action = UserMessage(
+                action, out_channel, sender_id, input_channel=self.name()
+            )
+            await on_new_message(context_action)
+        except Exception as e:
+            logger.error("Exception when trying to handle message.{0}".format(e))
+            logger.debug(e, exc_info=True)
+
+    def blueprint(self, on_new_message: Callable[[UserMessage], Awaitable[None]]):
         mattermost_webhook = Blueprint("mattermost_webhook", __name__)
 
         @mattermost_webhook.route("/", methods=["GET"])
@@ -86,27 +214,18 @@ async def health(request: Request):
         @mattermost_webhook.route("/webhook", methods=["POST"])
         async def webhook(request: Request):
             output = request.json
-            if output:
-                # splitting to get rid of the @botmention
-                # trigger we are using for this
-                text = output["text"].split(" ", 1)
-                text = text[1]
-                sender_id = output["user_id"]
-                self.bot_channel = output["channel_id"]
-                try:
-                    out_channel = MattermostBot(
-                        self.url, self.team, self.user, self.pw, self.bot_channel
-                    )
-                    user_msg = UserMessage(
-                        text, out_channel, sender_id, input_channel=self.name()
-                    )
-                    await on_new_message(user_msg)
-                except Exception as e:
-                    logger.error(
-                        "Exception when trying to handle message.{0}".format(e)
-                    )
-                    logger.debug(e, exc_info=True)
-                    pass
-            return response.text("")
+
+            if not output:
+                return response.text("")
+
+            # handle normal message with trigger_word
+            if "trigger_word" in output:
+                await self.message_with_trigger_word(on_new_message, output)
+
+            # handle context actions from buttons
+            elif "context" in output:
+                await self.action_from_button(on_new_message, output)
+
+            return response.text("success")
 
         return mattermost_webhook

rasa/core/channels/telegram.py

@@ -1,4 +1,5 @@
 import logging
+from copy import deepcopy
 from sanic import Blueprint, response
 from sanic.request import Request
 from telegram import (
@@ -94,6 +95,8 @@ async def send_text_with_buttons(
     async def send_custom_json(
         self, recipient_id: Text, json_message: Dict[Text, Any], **kwargs: Any
     ) -> None:
+        json_message = deepcopy(json_message)
+
         recipient_id = json_message.pop("chat_id", recipient_id)
 
         send_functions = {