Merge pull request #1100 from RasaHQ/utf8

Use utf8 everywhere for file IO. fixes #1041

CHANGELOG.rst

@@ -29,6 +29,7 @@ Fixed
 -----
 - fixed an issue with boolean slots where False and None had the same value
   (breaking model compatibility with models that use a boolean slot)
+- use utf8 everywhere when handling file IO
 - argument ``--connector`` on run script accepts custom channel module names
 - properly handle non ascii categorical slot values, e.g. ``大于100亿元``
 

rasa_core/domain.py

@@ -430,9 +430,8 @@ def load_specification(cls, path):
         # type: (Text) -> Dict[Text, Any]
         """Load a domains specification from a dumped model directory."""
 
-        matadata_path = os.path.join(path, 'domain.json')
-        with io.open(matadata_path) as f:
-            specification = json.loads(f.read())
+        metadata_path = os.path.join(path, 'domain.json')
+        specification = json.loads(utils.read_file(metadata_path))
         return specification
 
     def compare_with_specification(self, path):

rasa_core/evaluate.py

@@ -183,7 +183,7 @@ def log_failed_stories(failed, failed_output):
     if not failed_output:
         return
 
-    with io.open(failed_output, 'w') as f:
+    with io.open(failed_output, 'w', encoding="utf-8") as f:
         if len(failed) == 0:
             f.write("<!-- All stories passed -->")
         else:

rasa_core/featurizers.py

@@ -430,17 +430,15 @@ def create_X(self,
     def persist(self, path):
         featurizer_file = os.path.join(path, "featurizer.json")
         utils.create_dir_for_file(featurizer_file)
-        with io.open(featurizer_file, 'w') as f:
+        with io.open(featurizer_file, 'w', encoding="utf-8") as f:
             # noinspection PyTypeChecker
             f.write(str(jsonpickle.encode(self)))
 
     @staticmethod
     def load(path):
         featurizer_file = os.path.join(path, "featurizer.json")
         if os.path.isfile(featurizer_file):
-            with io.open(featurizer_file, 'r') as f:
-                _json = f.read()
-            return jsonpickle.decode(_json)
+            return jsonpickle.decode(utils.read_file(featurizer_file))
         else:
             logger.error("Couldn't load featurizer for policy. "
                          "File '{}' doesn't exist.".format(featurizer_file))

rasa_core/policies/ensemble.py

@@ -147,8 +147,7 @@ def persist(self, path, dump_flattened_stories=False):
     @classmethod
     def load_metadata(cls, path):
         metadata_path = os.path.join(path, 'policy_metadata.json')
-        with io.open(os.path.abspath(metadata_path)) as f:
-            metadata = json.loads(f.read())
+        metadata = json.loads(utils.read_file(os.path.abspath(metadata_path)))
         return metadata
 
     @staticmethod

rasa_core/policies/fallback.py

@@ -142,7 +142,6 @@ def load(cls, path):
         if os.path.exists(path):
             meta_path = os.path.join(path, "fallback_policy.json")
             if os.path.isfile(meta_path):
-                with io.open(meta_path) as f:
-                    meta = json.loads(f.read())
+                meta = json.loads(utils.read_file(meta_path))
 
         return cls(**meta)

rasa_core/policies/keras_policy.py

@@ -228,8 +228,7 @@ def load(cls, path):
             featurizer = TrackerFeaturizer.load(path)
             meta_path = os.path.join(path, "keras_policy.json")
             if os.path.isfile(meta_path):
-                with io.open(meta_path) as f:
-                    meta = json.loads(f.read())
+                meta = json.loads(utils.read_file(meta_path))
 
                 model_file = os.path.join(path, meta["model"])
 

rasa_core/policies/memoization.py

@@ -232,8 +232,7 @@ def load(cls, path):
         featurizer = TrackerFeaturizer.load(path)
         memorized_file = os.path.join(path, 'memorized_turns.json')
         if os.path.isfile(memorized_file):
-            with io.open(memorized_file) as f:
-                data = json.loads(f.read())
+            data = json.loads(utils.read_file(memorized_file))
             return cls(featurizer=featurizer, lookup=data["lookup"])
         else:
             logger.info("Couldn't load memoization for policy. "

rasa_core/trackers.py

@@ -338,7 +338,7 @@ def export_stories_to_file(self, export_path="debug.md"):
         # type: (Text) -> None
         """Dump the tracker as a story to a file."""
 
-        with io.open(export_path, 'a') as f:
+        with io.open(export_path, 'a', encoding="utf-8") as f:
             f.write(self.export_stories() + "\n")
 
     ###

rasa_core/training/dsl.py

@@ -148,7 +148,7 @@ def read_from_file(filename, domain, interpreter=RegexInterpreter(),
         """Given a md file reads the contained stories."""
 
         try:
-            with io.open(filename, "r") as f:
+            with io.open(filename, "r", encoding="utf-8") as f:
                 lines = f.readlines()
             reader = StoryFileReader(domain, interpreter, template_variables)
             return reader.process_lines(lines)

rasa_core/training/interactive.py

@@ -517,7 +517,7 @@ def _request_export_stories_info():
 
     def validate_path(path):
         try:
-            with io.open(path, "a"):
+            with io.open(path, "a", encoding="utf-8"):
                 return True
         except Exception as e:
             return "Failed to open file. {}".format(e)
@@ -570,7 +570,7 @@ def _write_stories_to_file(export_file_path, sender_id, endpoint):
 
     sub_conversations = _split_conversation_at_restarts(evts)
 
-    with io.open(export_file_path, 'a') as f:
+    with io.open(export_file_path, 'a', encoding="utf-8") as f:
         for conversation in sub_conversations:
             parsed_events = events.deserialise_events(conversation)
             s = Story.from_events(parsed_events)

rasa_core/training/structures.py

@@ -202,7 +202,7 @@ def as_story_string(self, flat=False):
             return story_content
 
     def dump_to_file(self, filename, flat=False):
-        with io.open(filename, "a") as f:
+        with io.open(filename, "a", encoding="utf-8") as f:
             f.write(self.as_story_string(flat))
 
 

rasa_core/utils.py

@@ -107,7 +107,7 @@ def dump_obj_as_str_to_file(filename, text):
     # type: (Text, Text) -> None
     """Dump a text to a file."""
 
-    with io.open(filename, 'w') as f:
+    with io.open(filename, 'w', encoding="utf-8") as f:
         # noinspection PyTypeChecker
         f.write(str(text))
 
@@ -547,7 +547,7 @@ def read_lines(filename, max_line_limit=None, line_pattern=".*"):
 
     line_filter = re.compile(line_pattern)
 
-    with io.open(filename, 'r') as f:
+    with io.open(filename, 'r', encoding="utf-8") as f:
         num_messages = 0
         for line in f:
             m = line_filter.match(line)

tests/test_dialogues.py

@@ -10,16 +10,15 @@
 import jsonpickle
 import pytest
 
+from rasa_core import utils
 from rasa_core.domain import Domain
 from rasa_core.tracker_store import InMemoryTrackerStore
 from tests.utilities import tracker_from_dialogue_file
 
 
 @pytest.mark.parametrize("filename", glob.glob('data/test_dialogues/*json'))
 def test_dialogue_serialisation(filename):
-    print("testing file: {0}".format(filename))
-    with io.open(filename, "r") as f:
-        dialogue_json = f.read()
+    dialogue_json = utils.read_file(filename)
     restored = json.loads(dialogue_json)
     tracker = tracker_from_dialogue_file(filename)
     en_de_coded = json.loads(jsonpickle.encode(tracker.as_dialogue()))

tests/test_dsl.py

@@ -61,7 +61,7 @@ def test_persist_and_read_test_story_graph(tmpdir, default_domain):
     graph = training.extract_story_graph("data/test_stories/stories.md",
                                          default_domain)
     out_path = tmpdir.join("persisted_story.md")
-    with io.open(out_path.strpath, "w") as f:
+    with io.open(out_path.strpath, "w", encoding="utf-8") as f:
         f.write(graph.as_story_string())
 
     recovered_trackers = training.load_data(

tests/utilities.py

@@ -12,6 +12,7 @@
 import jsonpickle
 import six
 
+from rasa_core import utils
 from rasa_core.domain import Domain
 from rasa_core.trackers import DialogueStateTracker
 from tests.conftest import DEFAULT_DOMAIN_PATH
@@ -30,14 +31,12 @@ def tracker_from_dialogue_file(filename, domain=None):
 
 
 def read_dialogue_file(filename):
-    with io.open(filename, "r") as f:
-        dialogue_json = f.read()
-    return jsonpickle.loads(dialogue_json)
+    return jsonpickle.loads(utils.read_file(filename))
 
 
 def write_text_to_file(tmpdir, filename, text):
     path = tmpdir.join(filename).strpath
-    with io.open(path, "w") as f:
+    with io.open(path, "w", encoding="utf-8") as f:
         f.write(text)
     return path