Merge pull request #1538 from RasaHQ/emojis_server

handle emojis when running as server

CHANGELOG.rst

@@ -21,6 +21,7 @@ Changed
 - the http server's ``POST /evaluate`` endpoint returns evaluation results
   for both entities and intents
 - use cloudpickle version 0.6.1
+- replaced ``yaml`` with ``ruamel.yaml``
 
 Removed
 -------
@@ -30,6 +31,7 @@ Fixed
 -----
 - Should loading jieba custom dictionaries only once.
 - Set attributes of custom components correctly if they defer from the default
+- NLU Server can now handle training data mit emojis in it
 
 [0.13.7] - 2018-10-11
 ^^^^^^^^^^^^^^^^^^^^^

alt_requirements/requirements_bare.txt

@@ -13,5 +13,5 @@ simplejson==3.13.2
 cloudpickle==0.6.1
 msgpack-python==0.5.4
 packaging==17.1
-pyyaml==3.12
+ruamel.yaml==0.15.78
 coloredlogs==9.0

data/examples/rasa/demo-rasa.json

@@ -22,12 +22,12 @@
     ],
     "common_examples": [
       {
-        "text": "hey", 
+        "text": "hey",
         "intent": "greet", 
         "entities": []
       }, 
       {
-        "text": "howdy", 
+        "text": "howdy",
         "intent": "greet", 
         "entities": []
       }, 

rasa_nlu/config.py

@@ -8,7 +8,7 @@
 import os
 
 import six
-import yaml
+import ruamel.yaml as yaml
 from builtins import object
 # Describes where to search for the config file if no location is specified
 from typing import Text, Optional, Dict, Any, List

rasa_nlu/utils/__init__.py

@@ -19,7 +19,7 @@
 import requests
 import simplejson
 import six
-import yaml
+import ruamel.yaml as yaml
 from builtins import str
 from future.utils import PY3
 from requests.auth import HTTPBasicAuth
@@ -243,13 +243,18 @@ def env_var_constructor(loader, node):
         prefix, env_var, postfix = env_var_pattern.match(value).groups()
         return prefix + os.environ[env_var] + postfix
 
-    yaml.add_constructor(u'!env_var', env_var_constructor)
+    yaml.SafeConstructor.add_constructor(u'!env_var', env_var_constructor)
 
 
 def read_yaml(content):
     fix_yaml_loader()
     replace_environment_variables()
-    return yaml.load(content)
+
+    yaml_parser = yaml.YAML(typ="safe")
+    yaml_parser.version = "1.2"
+    yaml_parser.unicode_supplementary = True
+
+    return yaml_parser.load(content)
 
 
 def read_yaml_file(filename):
@@ -357,8 +362,9 @@ def create_temporary_file(data, suffix="", mode="w+"):
     mode defines NamedTemporaryFile's  mode parameter in py3."""
 
     if PY3:
+        encoding = None if 'b' in mode else 'utf-8'
         f = tempfile.NamedTemporaryFile(mode=mode, suffix=suffix,
-                                        delete=False)
+                                        delete=False, encoding=encoding)
         f.write(data)
     else:
         f = tempfile.NamedTemporaryFile("w+", suffix=suffix,

setup.py

@@ -39,7 +39,7 @@
     "matplotlib~=2.0",
     "numpy>=1.13",
     "simplejson",
-    "pyyaml",
+    "ruamel.yaml~=0.15",
     'coloredlogs',
 ]
 

tests/base/test_server.py

@@ -10,7 +10,7 @@
 
 import pytest
 import time
-import yaml
+import ruamel.yaml as yaml
 from treq.testing import StubTreq
 
 from rasa_nlu.data_router import DataRouter

tests/base/test_utils.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -194,3 +195,44 @@ def test_environment_variable_dict_with_prefix_and_with_postfix():
     result = utils.read_yaml(content)
 
     assert result['model']['test'] == 'dir/test/dir'
+
+
+def test_emojis_in_yaml():
+    test_data = """
+    data:
+        - one 😁
+        - two £
+    """
+    actual = utils.read_yaml(test_data)
+
+    assert actual["data"][0] == "one 😁"
+    assert actual["data"][1] == "two £"
+
+
+def test_emojis_in_tmp_file():
+    test_data = """
+        data:
+            - one 😁
+            - two £
+        """
+    test_file = utils.create_temporary_file(test_data)
+    with io.open(test_file, mode='r', encoding="utf-8") as f:
+        content = f.read()
+    actual = utils.read_yaml(content)
+
+    assert actual["data"][0] == "one 😁"
+    assert actual["data"][1] == "two £"
+
+
+def test_bool_str():
+    test_data = """
+    one: "yes"
+    two: "true"
+    three: "True"
+    """
+
+    actual = utils.read_yaml(test_data)
+
+    assert actual["one"] == "yes"
+    assert actual["two"] == "true"
+    assert actual["three"] == "True"

tests/utilities.py

@@ -6,7 +6,7 @@
 import tempfile
 
 import pytest
-import yaml
+import ruamel.yaml as yaml
 from builtins import object
 
 from rasa_nlu.config import RasaNLUModelConfig