Support custom tags in YAML format

Extend the YAML parser in order to include any translatable content that
is marked with a custom type using tags.

openformats/formats/yaml/utils.py

@@ -67,6 +67,19 @@ def compose_node(self, parent, index):
             )
         return super(TxYamlLoader, self).compose_node(parent, index)
 
+    def _is_custom_tag(self, tag):
+        """
+        Check whether a value is tagged with a custom type.
+
+        Support custom tags, like: `foo: !bar test` Do not match `!!`, since it
+        indicates the built-in types. Match tags in the `!tag1!tag2!tag3` form.
+        """
+        # XXX: The  PyYAML library fails to handle multiple tags, like
+        # !tag1!tag2. There seems to be no workaround at the moment.
+        # XXX: We can't preserve the information whether the built-in `!!str`
+        # tag was used for a string
+        return re.match(r'^[\![a-zA-Z_]*]*$', tag, re.IGNORECASE)
+
     def construct_mapping(self, node, deep=True):
         """
         Override `yaml.SafeLoader.construct_mapping` to return for each item
@@ -94,13 +107,16 @@ def construct_mapping(self, node, deep=True):
                 hash(key)
             except TypeError as e:
                 print("Error while constructing a mapping, found unacceptable"
-                      " key (%s)".format(unicode(e)))
+                      " key ({})".format(unicode(e)))
                 continue
 
             if not(isinstance(value, unicode) or isinstance(value, str) or
                     isinstance(value, list) or isinstance(value, dict)):
                 continue
             start = value_node.start_mark.index
+            if self._is_custom_tag(value_node.tag):
+                # + 1 stands for the whitespace
+                start += len(value_node.tag) + 1
             end = value_node.end_mark.index
             style = ''
 

openformats/formats/yaml/yaml.py

@@ -4,6 +4,7 @@
 import copy
 from StringIO import StringIO
 from yaml.emitter import Emitter
+from yaml.constructor import SafeConstructor
 
 from openformats.handlers import Handler
 from openformats.strings import OpenString
@@ -64,6 +65,8 @@ def parse(self, content, **kwargs):
         template = ""
         context = ""
         stringset = []
+        SafeConstructor.add_constructor(None,
+                                        SafeConstructor.construct_yaml_str)
         yaml_data = self._load_yaml(content, loader=TxYamlLoader)
         yaml_data = self._get_yaml_data_to_parse(yaml_data)
         parsed_data = self._parse_yaml_data(yaml_data, '', [], context)
@@ -86,8 +89,11 @@ def parse(self, content, **kwargs):
             )
             stringset.append(string_object)
             order += 1
-            template += (content[end:start] +
-                         string_object.template_replacement)
+            template = "{template}{prefix}{txhash}".format(
+                template=template,
+                prefix=content[end:start],
+                txhash=string_object.template_replacement,
+            )
             comment = self._find_comment(content, end, start)
             string_object.developer_comment = comment
             end = end_

openformats/tests/formats/yaml/files/1_el.yml

@@ -69,3 +69,10 @@ anchor_key: &an_anchor
 alias_key:
   - *an_anchor
   - "el:another true value"
+
+# Custom tags
+foo:  !test "el:bar" # Should treat as string and ignore leading spaces
+bar: !xml "el:foo <xml>bar</xml>" # Also a string
+hello: "el:World" # Translatable
+number: !!int 123 # Should ignore
+bin: !!binary aGVsbG8= # Should ignore

openformats/tests/formats/yaml/files/1_en.yml

@@ -72,3 +72,10 @@ anchor_key: &an_anchor
 alias_key:
   - *an_anchor
   - "another true value"
+
+# Custom tags
+foo:  !test bar # Should treat as string and ignore leading spaces
+bar: !xml "foo <xml>bar</xml>" # Also a string
+hello: !!str World # Translatable
+number: !!int 123 # Should ignore
+bin: !!binary aGVsbG8= # Should ignore

openformats/tests/formats/yaml/files/1_en_exported.yml

@@ -69,3 +69,10 @@ anchor_key: &an_anchor
 alias_key:
   - *an_anchor
   - "another true value"
+
+# Custom tags
+foo:  !test bar # Should treat as string and ignore leading spaces
+bar: !xml "foo <xml>bar</xml>" # Also a string
+hello: World # Translatable
+number: !!int 123 # Should ignore
+bin: !!binary aGVsbG8= # Should ignore

openformats/tests/formats/yaml/files/1_en_exported_without_template.yml

@@ -36,3 +36,6 @@ anchor_key:
   - value
 alias_key:
   - "another true value"
+foo: bar
+bar: "foo <xml>bar</xml>"
+hello: World

openformats/tests/formats/yaml/files/1_tpl.yml

@@ -63,3 +63,10 @@ anchor_key: &an_anchor
 alias_key:
   - *an_anchor
   - ddc3cfcedcf1686d9e3ba6b99a0d091b_tr
+
+# Custom tags
+foo:  !test d06ae22aee8cc2abd5abd5a87cd784be_tr # Should treat as string and ignore leading spaces
+bar: !xml 14f55225e56b68a96e5df6b2377fa5f1_tr # Also a string
+hello: b0ed9cf22c0a5186d1c5b483a910dd33_tr # Translatable
+number: !!int 123 # Should ignore
+bin: !!binary aGVsbG8= # Should ignore