merged 1.5.x

CHANGELOG.rst

@@ -17,6 +17,37 @@ This project adheres to `Semantic Versioning`_ starting with version 1.0.
 
 .. towncrier release notes start
 
+[1.5.2] - 2019-12-09
+^^^^^^^^^^^^^^^^^^^^
+
+Improvements
+------------
+- `#3684 <https://github.com/rasahq/rasa/issues/3684>`_: ``rasa interactive`` will skip the story visualization of training stories in case
+  there are more than 200 stories. Stories created during interactive learning will be
+  visualized as before.
+- `#4792 <https://github.com/rasahq/rasa/issues/4792>`_: The log level for SocketIO loggers, including ``websockets.protocol``, ``engineio.server``,
+  and ``socketio.server``, is now handled by the ``LOG_LEVEL_LIBRARIES`` environment variable,
+  where the default log level is ``ERROR``.
+- `#4873 <https://github.com/rasahq/rasa/issues/4873>`_: Updated all example bots and documentation to use the updated ``dispatcher.utter_message()`` method from `rasa-sdk==1.5.0`.
+
+Bugfixes
+--------
+- `#3684 <https://github.com/rasahq/rasa/issues/3684>`_: ``rasa interactive`` will not load training stories in case the visualization is
+  skipped.
+- `#4789 <https://github.com/rasahq/rasa/issues/4789>`_: Fixed error where spacy models where not found in the docker images.
+- `#4802 <https://github.com/rasahq/rasa/issues/4802>`_: Fixed unnecessary ``kwargs`` unpacking in ``rasa.test.test_core`` call in ``rasa.test.test`` function.
+- `#4898 <https://github.com/rasahq/rasa/issues/4898>`_: Training data files now get loaded in the same order (especially relevant to subdirectories) each time to ensure training consistency when using a random seed.
+- `#4918 <https://github.com/rasahq/rasa/issues/4918>`_: Locks for tickets in ``LockStore`` are immediately issued without a redundant
+  check for their availability.
+
+Improved Documentation
+----------------------
+- `#4844 <https://github.com/rasahq/rasa/issues/4844>`_: Added ``towncrier`` to automatically collect changelog entries.
+- `#4869 <https://github.com/rasahq/rasa/issues/4869>`_: Document the pipeline for ``pretrained_embeddings_convert`` in the pre-configured pipelines section.
+- `#4894 <https://github.com/rasahq/rasa/issues/4894>`_: ``Proactively Reaching Out to the User Using Actions`` now correctly links to the
+  endpoint specification.
+
+
 [1.5.1] - 2019-11-27
 ^^^^^^^^^^^^^^^^^^^^
 

rasa/core/lock.py

@@ -100,10 +100,8 @@ def last_issued(self) -> int:
         """
 
         ticket_number = self._ticket_number_for(-1)
-        if ticket_number is not None:
-            return ticket_number
 
-        return NO_TICKET_ISSUED
+        return ticket_number if ticket_number is not None else NO_TICKET_ISSUED
 
     @property
     def now_serving(self) -> Optional[int]:

rasa/core/lock_store.py

@@ -2,12 +2,12 @@
 import json
 import logging
 import os
-from typing import Text, Optional, Union, AsyncGenerator
 
 from async_generator import asynccontextmanager
+from typing import Text, Optional, AsyncGenerator
 
 from rasa.core.constants import DEFAULT_LOCK_LIFETIME
-from rasa.core.lock import TicketLock, NO_TICKET_ISSUED
+from rasa.core.lock import TicketLock
 from rasa.utils.endpoints import EndpointConfig
 
 logger = logging.getLogger(__name__)
@@ -28,18 +28,6 @@ class LockError(Exception):
     pass
 
 
-# noinspection PyUnresolvedReferences
-class TicketExistsError(Exception):
-    """Exception that is raised when an already-existing ticket for a conversation
-    has been issued.
-
-     Attributes:
-          message (str): explanation of which `conversation_id` raised the error
-    """
-
-    pass
-
-
 class LockStore:
     @staticmethod
     def find_lock_store(store: EndpointConfig = None) -> "LockStore":
@@ -93,7 +81,7 @@ def save_lock(self, lock: TicketLock) -> None:
         raise NotImplementedError
 
     def issue_ticket(
-        self, conversation_id: Text, lock_lifetime: Union[float, int] = LOCK_LIFETIME
+        self, conversation_id: Text, lock_lifetime: float = LOCK_LIFETIME
     ) -> int:
         """Issue new ticket with `lock_lifetime` for lock associated with
         `conversation_id`.
@@ -103,18 +91,6 @@ def issue_ticket(
 
         lock = self.get_or_create_lock(conversation_id)
         ticket = lock.issue_ticket(lock_lifetime)
-
-        while True:
-            try:
-                self.ensure_ticket_available(lock)
-                break
-            except TicketExistsError:
-                # issue a new ticket if current ticket number has been issued twice
-                logger.exception(
-                    "Ticket could not be issued. Issuing new ticket and retrying..."
-                )
-                ticket = lock.issue_ticket(lock_lifetime)
-
         self.save_lock(lock)
 
         return ticket
@@ -123,8 +99,8 @@ def issue_ticket(
     async def lock(
         self,
         conversation_id: Text,
-        lock_lifetime: int = LOCK_LIFETIME,
-        wait_time_in_seconds: Union[int, float] = 1,
+        lock_lifetime: float = LOCK_LIFETIME,
+        wait_time_in_seconds: float = 1,
     ) -> AsyncGenerator[TicketLock, None]:
         """Acquire lock with lifetime `lock_lifetime`for `conversation_id`.
 
@@ -143,10 +119,7 @@ async def lock(
             self.cleanup(conversation_id, ticket)
 
     async def _acquire_lock(
-        self,
-        conversation_id: Text,
-        ticket: int,
-        wait_time_in_seconds: Union[int, float],
+        self, conversation_id: Text, ticket: int, wait_time_in_seconds: float,
     ) -> TicketLock:
 
         while True:
@@ -162,17 +135,16 @@ async def _acquire_lock(
                 return lock
 
             logger.debug(
-                "Failed to acquire lock for conversation ID '{}'. Retrying..."
-                "".format(conversation_id)
+                f"Failed to acquire lock for conversation ID '{conversation_id}'. "
+                f"Retrying..."
             )
 
             # sleep and update lock
             await asyncio.sleep(wait_time_in_seconds)
             self.update_lock(conversation_id)
 
         raise LockError(
-            "Could not acquire lock for conversation_id '{}'."
-            "".format(conversation_id)
+            f"Could not acquire lock for conversation_id '{conversation_id}'."
         )
 
     def update_lock(self, conversation_id: Text) -> None:
@@ -229,28 +201,6 @@ def _log_deletion(conversation_id: Text, deletion_successful: bool) -> None:
         else:
             logger.debug(f"Could not delete lock for conversation '{conversation_id}'.")
 
-    def ensure_ticket_available(self, lock: TicketLock) -> None:
-        """Check for duplicate tickets issued for `lock`.
-
-        This function should be called before saving `lock`. Raises `TicketExistsError`
-        if the last issued ticket for `lock` does not match the last ticket issued
-        for a lock fetched from storage for `lock.conversation_id`. This indicates
-        that some other process has issued a ticket for `lock` in the meantime.
-        """
-
-        existing_lock = self.get_lock(lock.conversation_id)
-        if not existing_lock or existing_lock.last_issued == NO_TICKET_ISSUED:
-            # lock does not yet exist for conversation or no ticket has been issued
-            return
-
-        # raise if the last issued ticket number of `existing_lock` is not the same as
-        # that of the one being acquired
-        if existing_lock.last_issued != lock.last_issued:
-            raise TicketExistsError(
-                "Ticket '{}' already exists for conversation ID '{}'."
-                "".format(existing_lock.last_issued, lock.conversation_id)
-            )
-
 
 class RedisLockStore(LockStore):
     """Redis store for ticket locks."""

rasa/utils/common.py

@@ -68,6 +68,7 @@ def set_log_level(log_level: Optional[int] = None):
     update_tensorflow_log_level()
     update_asyncio_log_level()
     update_apscheduler_log_level()
+    update_socketio_log_level()
 
     os.environ[ENV_LOG_LEVEL] = logging.getLevelName(log_level)
 
@@ -87,6 +88,20 @@ def update_apscheduler_log_level() -> None:
         logging.getLogger(logger_name).propagate = False
 
 
+def update_socketio_log_level() -> None:
+    log_level = os.environ.get(ENV_LOG_LEVEL_LIBRARIES, DEFAULT_LOG_LEVEL_LIBRARIES)
+
+    socketio_loggers = [
+        "websockets.protocol",
+        "engineio.server",
+        "socketio.server",
+    ]
+
+    for logger_name in socketio_loggers:
+        logging.getLogger(logger_name).setLevel(log_level)
+        logging.getLogger(logger_name).propagate = False
+
+
 def update_tensorflow_log_level() -> None:
     """Set the log level of Tensorflow to the log level specified in the environment
     variable 'LOG_LEVEL_LIBRARIES'."""

rasa/version.py

@@ -1 +1,3 @@
+# this file will automatically be changed,
+# do not add anything but the version number here!
 __version__ = "1.6.0a1"

scripts/publish_gh_release_notes.py

@@ -4,7 +4,7 @@
 
 Uses the following environment variables:
 * TRAVIS_TAG: the name of the tag of the current commit.
-* GH_RELEASE_NOTES_TOKEN: a personal access token with 'repo' permissions.
+* GITHUB_TOKEN: a personal access token with 'repo' permissions.
 
 The script also requires ``pandoc`` to be previously installed in the system.
 Requires Python3.6+.
@@ -70,9 +70,9 @@ def main():
         print("environment variable TRAVIS_TAG not set", file=sys.stderr)
         return 1
 
-    token = os.environ.get("GH_RELEASE_NOTES_TOKEN")
+    token = os.environ.get("GITHUB_TOKEN")
     if not token:
-        print("GH_RELEASE_NOTES_TOKEN not set", file=sys.stderr)
+        print("GITHUB_TOKEN not set", file=sys.stderr)
         return 1
 
     slug = os.environ.get("TRAVIS_REPO_SLUG")

tests/core/test_lock_store.py

@@ -1,20 +1,36 @@
 import asyncio
-import copy
 import os
-from typing import Union, Text
-from unittest.mock import patch
 
 import numpy as np
 import pytest
 import time
 from _pytest.tmpdir import TempdirFactory
+from unittest.mock import patch
 
-import rasa.utils.io
 from rasa.core.agent import Agent
 from rasa.core.channels import UserMessage
 from rasa.core.constants import INTENT_MESSAGE_PREFIX, DEFAULT_LOCK_LIFETIME
-from rasa.core.lock import TicketLock, Ticket
-from rasa.core.lock_store import InMemoryLockStore, LockError, TicketExistsError
+from rasa.core.lock import TicketLock
+from rasa.core.lock_store import (
+    InMemoryLockStore,
+    LockError,
+    LockStore,
+    RedisLockStore,
+)
+
+
+class FakeRedisLockStore(RedisLockStore):
+    """Fake `RedisLockStore` using `fakeredis` library."""
+
+    def __init__(self):
+        import fakeredis
+
+        self.red = fakeredis.FakeStrictRedis()
+
+        # added in redis==3.3.0, but not yet in fakeredis
+        self.red.connection_pool.connection_class.health_check_interval = 0
+
+        super(RedisLockStore, self).__init__()
 
 
 def test_issue_ticket():
@@ -52,8 +68,8 @@ def test_remove_expired_tickets():
     assert len(lock.tickets) == 1
 
 
-def test_create_lock_store():
-    lock_store = InMemoryLockStore()
+@pytest.mark.parametrize("lock_store", [InMemoryLockStore(), FakeRedisLockStore()])
+def test_create_lock_store(lock_store: LockStore):
     conversation_id = "my id 0"
 
     # create and lock
@@ -64,8 +80,8 @@ def test_create_lock_store():
     assert lock.conversation_id == conversation_id
 
 
-def test_serve_ticket():
-    lock_store = InMemoryLockStore()
+@pytest.mark.parametrize("lock_store", [InMemoryLockStore(), FakeRedisLockStore()])
+def test_serve_ticket(lock_store: LockStore):
     conversation_id = "my id 1"
 
     lock = lock_store.create_lock(conversation_id)
@@ -99,8 +115,9 @@ def test_serve_ticket():
     assert not lock.is_someone_waiting()
 
 
-def test_lock_expiration():
-    lock_store = InMemoryLockStore()
+# noinspection PyProtectedMember
+@pytest.mark.parametrize("lock_store", [InMemoryLockStore(), FakeRedisLockStore()])
+def test_lock_expiration(lock_store: LockStore):
     conversation_id = "my id 2"
     lock = lock_store.create_lock(conversation_id)
     lock_store.save_lock(lock)
@@ -120,33 +137,6 @@ def test_lock_expiration():
     assert lock.issue_ticket(10) == 1
 
 
-def test_ticket_exists_error():
-    def mocked_issue_ticket(
-        self,
-        conversation_id: Text,
-        lock_lifetime: Union[float, int] = DEFAULT_LOCK_LIFETIME,
-    ) -> None:
-        # mock LockStore.issue_ticket() so it issues two tickets for the same
-        # conversation ID simultaneously
-
-        lock = self.get_or_create_lock(conversation_id)
-        lock.issue_ticket(lock_lifetime)
-        self.save_lock(lock)
-
-        # issue another ticket for this lock
-        lock_2 = copy.deepcopy(lock)
-        lock_2.tickets.append(Ticket(1, time.time() + DEFAULT_LOCK_LIFETIME))
-
-        self.ensure_ticket_available(lock_2)
-
-    lock_store = InMemoryLockStore()
-    conversation_id = "my id 3"
-
-    with patch.object(InMemoryLockStore, "issue_ticket", mocked_issue_ticket):
-        with pytest.raises(TicketExistsError):
-            lock_store.issue_ticket(conversation_id)
-
-
 async def test_multiple_conversation_ids(default_agent: Agent):
     text = INTENT_MESSAGE_PREFIX + 'greet{"name":"Rasa"}'
 
@@ -176,9 +166,7 @@ async def test_message_order(tmpdir_factory: TempdirFactory, default_agent: Agen
     # record messages as they come and and as they're processed in files so we
     # can check the order later on. We don't need the return value of this method so
     # we'll just return None.
-    async def mocked_handle_message(
-        self, message: UserMessage, wait: Union[int, float]
-    ) -> None:
+    async def mocked_handle_message(self, message: UserMessage, wait: float) -> None:
         # write incoming message to file
         with open(str(incoming_order_file), "a+") as f_0:
             f_0.write(message.text + "\n")