diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4a37649ef51..a4a34c4b84c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,6 +17,37 @@ This project adheres to `Semantic Versioning`_ starting with version 1.0. .. towncrier release notes start +[1.5.2] - 2019-12-09 +^^^^^^^^^^^^^^^^^^^^ + +Improvements +------------ +- `#3684 `_: ``rasa interactive`` will skip the story visualization of training stories in case + there are more than 200 stories. Stories created during interactive learning will be + visualized as before. +- `#4792 `_: The log level for SocketIO loggers, including ``websockets.protocol``, ``engineio.server``, + and ``socketio.server``, is now handled by the ``LOG_LEVEL_LIBRARIES`` environment variable, + where the default log level is ``ERROR``. +- `#4873 `_: Updated all example bots and documentation to use the updated ``dispatcher.utter_message()`` method from `rasa-sdk==1.5.0`. + +Bugfixes +-------- +- `#3684 `_: ``rasa interactive`` will not load training stories in case the visualization is + skipped. +- `#4789 `_: Fixed error where spacy models where not found in the docker images. +- `#4802 `_: Fixed unnecessary ``kwargs`` unpacking in ``rasa.test.test_core`` call in ``rasa.test.test`` function. +- `#4898 `_: Training data files now get loaded in the same order (especially relevant to subdirectories) each time to ensure training consistency when using a random seed. +- `#4918 `_: Locks for tickets in ``LockStore`` are immediately issued without a redundant + check for their availability. + +Improved Documentation +---------------------- +- `#4844 `_: Added ``towncrier`` to automatically collect changelog entries. +- `#4869 `_: Document the pipeline for ``pretrained_embeddings_convert`` in the pre-configured pipelines section. +- `#4894 `_: ``Proactively Reaching Out to the User Using Actions`` now correctly links to the + endpoint specification. + + [1.5.1] - 2019-11-27 ^^^^^^^^^^^^^^^^^^^^ diff --git a/changelog/3684.bugfix.rst b/changelog/3684.bugfix.rst deleted file mode 100644 index bbad285950d..00000000000 --- a/changelog/3684.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -``rasa interactive`` will not load training stories in case the visualization is -skipped. diff --git a/changelog/3684.improvement.rst b/changelog/3684.improvement.rst deleted file mode 100644 index 4cd23fa9296..00000000000 --- a/changelog/3684.improvement.rst +++ /dev/null @@ -1,3 +0,0 @@ -``rasa interactive`` will skip the story visualization of training stories in case -there are more than 200 stories. Stories created duringinteractive learning will be -visualized as before. diff --git a/changelog/4789.bugfix.rst b/changelog/4789.bugfix.rst deleted file mode 100644 index 24969d95d3e..00000000000 --- a/changelog/4789.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed error where spacy models where not found in the docker images. \ No newline at end of file diff --git a/changelog/4802.bugfix.rst b/changelog/4802.bugfix.rst deleted file mode 100644 index 0df1b68c952..00000000000 --- a/changelog/4802.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed unnecessary ``kwargs`` unpacking in ``rasa.test.test_core`` call in ``rasa.test.test`` function. diff --git a/changelog/4844.doc.rst b/changelog/4844.doc.rst deleted file mode 100644 index d81ba0f3dd1..00000000000 --- a/changelog/4844.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Added ``towncrier`` to automatically collect changelog entries. \ No newline at end of file diff --git a/changelog/4844.misc.rst b/changelog/4844.misc.rst deleted file mode 100644 index 80d672ee6ee..00000000000 --- a/changelog/4844.misc.rst +++ /dev/null @@ -1,4 +0,0 @@ -Added ``release`` command to makefile. - -The release command will automatically prepare all necessary -things (changelog, version file, ...) for the next release. diff --git a/changelog/4869.doc.rst b/changelog/4869.doc.rst deleted file mode 100644 index 5a3a6a2103d..00000000000 --- a/changelog/4869.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Document the pipeline for ``pretrained_embeddings_convert`` in the pre-configured pipelines section. diff --git a/changelog/4873.improvement.rst b/changelog/4873.improvement.rst deleted file mode 100644 index fd437dbd355..00000000000 --- a/changelog/4873.improvement.rst +++ /dev/null @@ -1 +0,0 @@ -Updated all example bots and documentation to use the updated ``dispatcher.utter_message()`` method from `rasa-sdk==1.5.0`. diff --git a/changelog/4894.doc.rst b/changelog/4894.doc.rst deleted file mode 100644 index cacd4f4dbf0..00000000000 --- a/changelog/4894.doc.rst +++ /dev/null @@ -1,2 +0,0 @@ -``Proactively Reaching Out to the User Using Actions`` now correctly links to the -endpoint specification. \ No newline at end of file diff --git a/changelog/4898.bugfix.rst b/changelog/4898.bugfix.rst deleted file mode 100644 index f4787c86ad5..00000000000 --- a/changelog/4898.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Training data files now get loaded in the same order (especially relevant to subdirectories) each time to ensure training consistency when using a random seed. diff --git a/rasa/core/lock.py b/rasa/core/lock.py index 8c01a85848e..5588ab3f2b7 100644 --- a/rasa/core/lock.py +++ b/rasa/core/lock.py @@ -100,10 +100,8 @@ def last_issued(self) -> int: """ ticket_number = self._ticket_number_for(-1) - if ticket_number is not None: - return ticket_number - return NO_TICKET_ISSUED + return ticket_number if ticket_number is not None else NO_TICKET_ISSUED @property def now_serving(self) -> Optional[int]: diff --git a/rasa/core/lock_store.py b/rasa/core/lock_store.py index 7d2458abd77..1a7be3a00f3 100644 --- a/rasa/core/lock_store.py +++ b/rasa/core/lock_store.py @@ -2,12 +2,12 @@ import json import logging import os -from typing import Text, Optional, Union, AsyncGenerator from async_generator import asynccontextmanager +from typing import Text, Optional, AsyncGenerator from rasa.core.constants import DEFAULT_LOCK_LIFETIME -from rasa.core.lock import TicketLock, NO_TICKET_ISSUED +from rasa.core.lock import TicketLock from rasa.utils.endpoints import EndpointConfig logger = logging.getLogger(__name__) @@ -28,18 +28,6 @@ class LockError(Exception): pass -# noinspection PyUnresolvedReferences -class TicketExistsError(Exception): - """Exception that is raised when an already-existing ticket for a conversation - has been issued. - - Attributes: - message (str): explanation of which `conversation_id` raised the error - """ - - pass - - class LockStore: @staticmethod def find_lock_store(store: EndpointConfig = None) -> "LockStore": @@ -93,7 +81,7 @@ def save_lock(self, lock: TicketLock) -> None: raise NotImplementedError def issue_ticket( - self, conversation_id: Text, lock_lifetime: Union[float, int] = LOCK_LIFETIME + self, conversation_id: Text, lock_lifetime: float = LOCK_LIFETIME ) -> int: """Issue new ticket with `lock_lifetime` for lock associated with `conversation_id`. @@ -103,18 +91,6 @@ def issue_ticket( lock = self.get_or_create_lock(conversation_id) ticket = lock.issue_ticket(lock_lifetime) - - while True: - try: - self.ensure_ticket_available(lock) - break - except TicketExistsError: - # issue a new ticket if current ticket number has been issued twice - logger.exception( - "Ticket could not be issued. Issuing new ticket and retrying..." - ) - ticket = lock.issue_ticket(lock_lifetime) - self.save_lock(lock) return ticket @@ -123,8 +99,8 @@ def issue_ticket( async def lock( self, conversation_id: Text, - lock_lifetime: int = LOCK_LIFETIME, - wait_time_in_seconds: Union[int, float] = 1, + lock_lifetime: float = LOCK_LIFETIME, + wait_time_in_seconds: float = 1, ) -> AsyncGenerator[TicketLock, None]: """Acquire lock with lifetime `lock_lifetime`for `conversation_id`. @@ -143,10 +119,7 @@ async def lock( self.cleanup(conversation_id, ticket) async def _acquire_lock( - self, - conversation_id: Text, - ticket: int, - wait_time_in_seconds: Union[int, float], + self, conversation_id: Text, ticket: int, wait_time_in_seconds: float, ) -> TicketLock: while True: @@ -162,8 +135,8 @@ async def _acquire_lock( return lock logger.debug( - "Failed to acquire lock for conversation ID '{}'. Retrying..." - "".format(conversation_id) + f"Failed to acquire lock for conversation ID '{conversation_id}'. " + f"Retrying..." ) # sleep and update lock @@ -171,8 +144,7 @@ async def _acquire_lock( self.update_lock(conversation_id) raise LockError( - "Could not acquire lock for conversation_id '{}'." - "".format(conversation_id) + f"Could not acquire lock for conversation_id '{conversation_id}'." ) def update_lock(self, conversation_id: Text) -> None: @@ -229,28 +201,6 @@ def _log_deletion(conversation_id: Text, deletion_successful: bool) -> None: else: logger.debug(f"Could not delete lock for conversation '{conversation_id}'.") - def ensure_ticket_available(self, lock: TicketLock) -> None: - """Check for duplicate tickets issued for `lock`. - - This function should be called before saving `lock`. Raises `TicketExistsError` - if the last issued ticket for `lock` does not match the last ticket issued - for a lock fetched from storage for `lock.conversation_id`. This indicates - that some other process has issued a ticket for `lock` in the meantime. - """ - - existing_lock = self.get_lock(lock.conversation_id) - if not existing_lock or existing_lock.last_issued == NO_TICKET_ISSUED: - # lock does not yet exist for conversation or no ticket has been issued - return - - # raise if the last issued ticket number of `existing_lock` is not the same as - # that of the one being acquired - if existing_lock.last_issued != lock.last_issued: - raise TicketExistsError( - "Ticket '{}' already exists for conversation ID '{}'." - "".format(existing_lock.last_issued, lock.conversation_id) - ) - class RedisLockStore(LockStore): """Redis store for ticket locks.""" diff --git a/rasa/utils/common.py b/rasa/utils/common.py index 08bbce97842..347a029b9be 100644 --- a/rasa/utils/common.py +++ b/rasa/utils/common.py @@ -68,6 +68,7 @@ def set_log_level(log_level: Optional[int] = None): update_tensorflow_log_level() update_asyncio_log_level() update_apscheduler_log_level() + update_socketio_log_level() os.environ[ENV_LOG_LEVEL] = logging.getLevelName(log_level) @@ -87,6 +88,20 @@ def update_apscheduler_log_level() -> None: logging.getLogger(logger_name).propagate = False +def update_socketio_log_level() -> None: + log_level = os.environ.get(ENV_LOG_LEVEL_LIBRARIES, DEFAULT_LOG_LEVEL_LIBRARIES) + + socketio_loggers = [ + "websockets.protocol", + "engineio.server", + "socketio.server", + ] + + for logger_name in socketio_loggers: + logging.getLogger(logger_name).setLevel(log_level) + logging.getLogger(logger_name).propagate = False + + def update_tensorflow_log_level() -> None: """Set the log level of Tensorflow to the log level specified in the environment variable 'LOG_LEVEL_LIBRARIES'.""" diff --git a/rasa/version.py b/rasa/version.py index d416ae5e47f..12171a743b4 100644 --- a/rasa/version.py +++ b/rasa/version.py @@ -1 +1,3 @@ +# this file will automatically be changed, +# do not add anything but the version number here! __version__ = "1.6.0a1" diff --git a/scripts/publish_gh_release_notes.py b/scripts/publish_gh_release_notes.py index 50b58bf55e8..b029fc0429b 100644 --- a/scripts/publish_gh_release_notes.py +++ b/scripts/publish_gh_release_notes.py @@ -4,7 +4,7 @@ Uses the following environment variables: * TRAVIS_TAG: the name of the tag of the current commit. -* GH_RELEASE_NOTES_TOKEN: a personal access token with 'repo' permissions. +* GITHUB_TOKEN: a personal access token with 'repo' permissions. The script also requires ``pandoc`` to be previously installed in the system. Requires Python3.6+. @@ -70,9 +70,9 @@ def main(): print("environment variable TRAVIS_TAG not set", file=sys.stderr) return 1 - token = os.environ.get("GH_RELEASE_NOTES_TOKEN") + token = os.environ.get("GITHUB_TOKEN") if not token: - print("GH_RELEASE_NOTES_TOKEN not set", file=sys.stderr) + print("GITHUB_TOKEN not set", file=sys.stderr) return 1 slug = os.environ.get("TRAVIS_REPO_SLUG") diff --git a/tests/core/test_lock_store.py b/tests/core/test_lock_store.py index 47989a23bd9..ff3f83a879d 100644 --- a/tests/core/test_lock_store.py +++ b/tests/core/test_lock_store.py @@ -1,20 +1,36 @@ import asyncio -import copy import os -from typing import Union, Text -from unittest.mock import patch import numpy as np import pytest import time from _pytest.tmpdir import TempdirFactory +from unittest.mock import patch -import rasa.utils.io from rasa.core.agent import Agent from rasa.core.channels import UserMessage from rasa.core.constants import INTENT_MESSAGE_PREFIX, DEFAULT_LOCK_LIFETIME -from rasa.core.lock import TicketLock, Ticket -from rasa.core.lock_store import InMemoryLockStore, LockError, TicketExistsError +from rasa.core.lock import TicketLock +from rasa.core.lock_store import ( + InMemoryLockStore, + LockError, + LockStore, + RedisLockStore, +) + + +class FakeRedisLockStore(RedisLockStore): + """Fake `RedisLockStore` using `fakeredis` library.""" + + def __init__(self): + import fakeredis + + self.red = fakeredis.FakeStrictRedis() + + # added in redis==3.3.0, but not yet in fakeredis + self.red.connection_pool.connection_class.health_check_interval = 0 + + super(RedisLockStore, self).__init__() def test_issue_ticket(): @@ -52,8 +68,8 @@ def test_remove_expired_tickets(): assert len(lock.tickets) == 1 -def test_create_lock_store(): - lock_store = InMemoryLockStore() +@pytest.mark.parametrize("lock_store", [InMemoryLockStore(), FakeRedisLockStore()]) +def test_create_lock_store(lock_store: LockStore): conversation_id = "my id 0" # create and lock @@ -64,8 +80,8 @@ def test_create_lock_store(): assert lock.conversation_id == conversation_id -def test_serve_ticket(): - lock_store = InMemoryLockStore() +@pytest.mark.parametrize("lock_store", [InMemoryLockStore(), FakeRedisLockStore()]) +def test_serve_ticket(lock_store: LockStore): conversation_id = "my id 1" lock = lock_store.create_lock(conversation_id) @@ -99,8 +115,9 @@ def test_serve_ticket(): assert not lock.is_someone_waiting() -def test_lock_expiration(): - lock_store = InMemoryLockStore() +# noinspection PyProtectedMember +@pytest.mark.parametrize("lock_store", [InMemoryLockStore(), FakeRedisLockStore()]) +def test_lock_expiration(lock_store: LockStore): conversation_id = "my id 2" lock = lock_store.create_lock(conversation_id) lock_store.save_lock(lock) @@ -120,33 +137,6 @@ def test_lock_expiration(): assert lock.issue_ticket(10) == 1 -def test_ticket_exists_error(): - def mocked_issue_ticket( - self, - conversation_id: Text, - lock_lifetime: Union[float, int] = DEFAULT_LOCK_LIFETIME, - ) -> None: - # mock LockStore.issue_ticket() so it issues two tickets for the same - # conversation ID simultaneously - - lock = self.get_or_create_lock(conversation_id) - lock.issue_ticket(lock_lifetime) - self.save_lock(lock) - - # issue another ticket for this lock - lock_2 = copy.deepcopy(lock) - lock_2.tickets.append(Ticket(1, time.time() + DEFAULT_LOCK_LIFETIME)) - - self.ensure_ticket_available(lock_2) - - lock_store = InMemoryLockStore() - conversation_id = "my id 3" - - with patch.object(InMemoryLockStore, "issue_ticket", mocked_issue_ticket): - with pytest.raises(TicketExistsError): - lock_store.issue_ticket(conversation_id) - - async def test_multiple_conversation_ids(default_agent: Agent): text = INTENT_MESSAGE_PREFIX + 'greet{"name":"Rasa"}' @@ -176,9 +166,7 @@ async def test_message_order(tmpdir_factory: TempdirFactory, default_agent: Agen # record messages as they come and and as they're processed in files so we # can check the order later on. We don't need the return value of this method so # we'll just return None. - async def mocked_handle_message( - self, message: UserMessage, wait: Union[int, float] - ) -> None: + async def mocked_handle_message(self, message: UserMessage, wait: float) -> None: # write incoming message to file with open(str(incoming_order_file), "a+") as f_0: f_0.write(message.text + "\n")