refactor: language as intent-level metadata, rest as example level

botfront · Dec 2, 2020 · a241685 · a241685
1 parent e42247b
commit a241685
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 11 deletions.
diff --git a/rasa/shared/nlu/training_data/formats/rasa_yaml.py b/rasa/shared/nlu/training_data/formats/rasa_yaml.py
@@ -474,12 +474,22 @@ def process_training_examples_by_key(
         result = []
         for entity_key, examples in training_examples.items():
 
-            converted_examples = [
-                TrainingDataWriter.generate_list_item(
-                    example_extraction_predicate(example).strip(STRIP_SYMBOLS)
-                )
-                for example in examples
-            ]
+            # bf >
+            # converted_examples = [
+            #     TrainingDataWriter.generate_list_item(
+            #         example_extraction_predicate(example).strip(STRIP_SYMBOLS)
+            #     )
+            #     for example in examples
+            # ]
+            converted_examples = []
+            for example in examples:
+                metadata = example.pop("metadata", {})
+                sorted_example = OrderedDict()
+                sorted_example["text"] = example_extraction_predicate(example)
+                if metadata:
+                    sorted_example["metadata"] = metadata
+                converted_examples.append(sorted_example)
+            # < bf
 
             next_item = OrderedDict()
             # bf >
@@ -488,12 +498,11 @@ def process_training_examples_by_key(
             metadata = {}
             if entity_key[1] is not None:
                 metadata["language"] = entity_key[1]
-            if entity_key[2] is True:
-                metadata["canonical"] = entity_key[2]
             if metadata:
                 next_item["metadata"] = metadata
+            # next_item[key_examples] = LiteralScalarString("".join(converted_examples))
+            next_item[key_examples] = converted_examples
             # < bf
-            next_item[key_examples] = LiteralScalarString("".join(converted_examples))
             result.append(next_item)
 
         return result
diff --git a/rasa/shared/nlu/training_data/formats/readerwriter.py b/rasa/shared/nlu/training_data/formats/readerwriter.py
@@ -62,13 +62,12 @@ def prepare_training_examples(training_data: "TrainingData") -> OrderedDict:
             # bf >
             metadata = example.pop("metadata", {})
             language = metadata.pop("language", None)
-            canonical = metadata.pop("canonical", False)
             if metadata:
                 example["metadata"] = metadata
             # < bf
             rasa_nlu_training_data_utils.remove_untrainable_entities_from(example)
             # bf >
-            intent = (example[INTENT], language, canonical)
+            intent = (example[INTENT], language)
             # intent = example[INTENT]
             # < bf
             training_examples.setdefault(intent, [])