Skip to content

Commit

Permalink
Include all translatable content in Stringsdict
Browse files Browse the repository at this point in the history 
Make the Stringsdict format parser transform the source file in order to
include all the translatable content in the produced strings
  • Loading branch information
Rigas Papathanasopoulos committed Apr 16, 2018
1 parent 000d216 commit c1c1c4e
Show file tree
Hide file tree
Showing 5 changed files with 233 additions and 6 deletions.
104 changes: 98 additions & 6 deletions openformats/formats/stringsdict.py
View file
@@ -1,6 +1,7 @@
from __future__ import absolute_import

import itertools
import re
from xml.sax import saxutils

from ..handlers import Handler
Expand All @@ -27,11 +28,14 @@ class StringsDictHandler(Handler):
# Where to start parsing the file
PARSE_START = "<dict"

# Ignored keys
# Ignored or special keys
KEY_FORMAT = 'NSStringLocalizedFormatKey'
KEY_SPEC = 'NSStringFormatSpecTypeKey'
KEY_VALUE = 'NSStringFormatValueTypeKey'

# Regular expression for handling content with text and variables mixed
VALUE_CONTENT_RE = r'(?P<prefix>[^@]*)%#@(?P<variable>\w+)@(?P<suffix>.*)'

# Placeholders
PLACEHOLDER_KEY_TAG = 'tx_awesome_key_tag'
PLACEHOLDER_KEY = '<tx_awesome_key_tag></tx_awesome_key_tag>'
Expand Down Expand Up @@ -80,6 +84,43 @@ def parse(self, content, **kwargs):
def _handle_child_pairs(self, key_tag, dict_tag):
"""Handles the <key> tag and its <dict> value tag.
Note that in order to avoid splitting strings we perform the following
inline-replacement:
<key>NSStringLocalizedFormatKey</key>
<string>Look! There %#@mouse@ there</string>
<key>mouse</key>
<dict>
<key>NSStringFormatSpecTypeKey</key>
<string>NSStringPluralRuleType</string>
<key>NSStringFormatValueTypeKey</key>
<string>d</string>
<key>one</key>
<string>is a mouse</string>
<key>other</key>
<string>are %d mice</string>
</dict>
Becomes:
<key>NSStringLocalizedFormatKey</key>
<string>%#@mouse@</string>
<key>mouse</key>
<dict>
<key>NSStringFormatSpecTypeKey</key>
<string>NSStringPluralRuleType</string>
<key>NSStringFormatValueTypeKey</key>
<string>d</string>
<key>one</key>
<string>Look! There is a mouse there</string>
<key>other</key>
<string>Look! There are %d mice there</string>
</dict>
This is necessary to avoid splitting sentences in Transifex, or omit
parts of the translatable content. We reference this inline-replacement
in the comments below as [1].
:param key_tag: The <key> tag to be handled.
:param dict_tag: The <dict> tag to be handled.
:returns: A list containing the openstrings created. If no strings were
Expand All @@ -90,34 +131,73 @@ def _handle_child_pairs(self, key_tag, dict_tag):
dict_iterator = self._handle_dict(dict_tag)

string_list = []
# A helper variable to save the prefix and suffix needed for the
# inline-replacement [1].
text_extras = None

for key_child in dict_iterator:
# The second key contains the secondary key.
secondary_key = self._handle_key(key_child)
value_tag = self._get_key_value(dict_iterator, key_child)
if secondary_key == self.KEY_FORMAT:
matches = re.match(self.VALUE_CONTENT_RE, value_tag.content)
if matches is not None:
# The prefix and the suffix are relative to the FIRST
# variable
text_extras = {
"variable": matches.group("variable"),
"prefix": matches.group("prefix"),
"suffix": matches.group("suffix"),
}
# If the key is the one of the stringsdict defaults skip it
continue

openstring = self._handle_strings(
value_tag,
main_key,
secondary_key
secondary_key,
text_extras,
)
if openstring is not None:
# If an openstring was created append it to the list
string_list.append(openstring)
return string_list

def _handle_strings(self, dict_tag, main_key, secondary_key):
def _handle_strings(self, dict_tag, main_key, secondary_key, extras):
"""Handles the <dict> tag that contains the strings.
:param dict_tag: The <dict> tag containing the strings.
:param main_key: The main key. Used as the openstring's name.
:param secondary_key: The secondary key. Used as the openstring's
context.
:param extras: A dictionary containing the `variable`, `prefix` and
`suffix` keys if we need to perform replacement [1],
`None` otherwise. This parameter is the same for each
call under the same main key.
"""
dict_iterator = self._handle_dict(dict_tag)
self.transcriber.copy_until(dict_tag.position)

# If inline-replacement [1] is required, the first step is to remove
# the prefix and the suffix from the original string.
if extras is not None:
# This isolates the part of the source file between the main key
# and the plural rules, so we only replace the value of the
# NSStringLocalizedFormatKey value
text = self.transcriber.source[
self.transcriber.ptr:dict_tag.position
]
text = text.replace('<string>{}'.format(extras['prefix']),
'<string>', 1)
text = text.replace('{}</string>'.format(extras['suffix']),
'</string>', 1)
self.transcriber.add(text)
self.transcriber.skip_until(dict_tag.position)
else:
self.transcriber.copy_until(dict_tag.position)

# Save the line in which the plural rule starts to associate it with
# the openstring object. This is used later for generating helpful
# error messages
line_number = self.transcriber.line_number
if dict_tag.text is not None:
self.transcriber.copy_until(
Expand All @@ -129,13 +209,25 @@ def _handle_strings(self, dict_tag, main_key, secondary_key):
key_content = self._handle_key(key_tag)
value_tag = self._get_key_value(dict_iterator, key_tag)

# Skip key-value pairs that don't include any translatable content
if key_content in [self.KEY_SPEC, self.KEY_VALUE]:
self.transcriber.copy_until(value_tag.tail_position)
continue

# Get rule number from the key content
rule_number = self._validate_plural(key_content, key_tag)
strings_dict[rule_number] = value_tag.content
# If inline-replacement [1] is needed, and we now process the FIRST
# variable, we apply the prefix and the suffix to each of the
# plural rules
if extras is not None and extras["variable"] == secondary_key:
content = "{prefix}{content}{suffix}".format(
prefix=extras["prefix"],
content=value_tag.content,
suffix=extras["suffix"],
)
else:
content = value_tag.content
strings_dict[rule_number] = content

# If empty <string> tag keep it. It is either a placeholder which
# should be kept or it is missing plurals and we will raise a
Expand Down Expand Up @@ -425,4 +517,4 @@ def escape(string):

@staticmethod
def unescape(string):
return saxutils.unescape(string)
return saxutils.unescape(string)
43 changes: 43 additions & 0 deletions openformats/tests/formats/stringsdict/files/1_el.stringsdict
View file
Expand Up @@ -45,5 +45,48 @@
<string>el:Other tests exist.</string>
</dict>
</dict>
<key>TEST INLINE REPLACEMENT</key>
<dict>
<key>NSStringLocalizedFormatKey</key>
<string>%#@test@</string>
<key>test</key>
<dict>
<key>NSStringFormatSpecTypeKey</key>
<string>NSStringPluralRuleType</string>
<key>NSStringFormatValueTypeKey</key>
<string>d</string>
<key>one</key>
<string>el:There is a mouse in the %#@room@ here</string>
<key>other</key>
<string>el:There are %d mice in the %#@room@ here</string>
</dict>
<key>room</key>
<dict>
<key>NSStringFormatSpecTypeKey</key>
<string>NSStringPluralRuleType</string>
<key>NSStringFormatValueTypeKey</key>
<string>d</string>
<key>one</key>
<string>el:a room</string>
<key>other</key>
<string>el:%d rooms</string>
</dict>
</dict>
<key>LAST</key>
<dict>
<key>NSStringLocalizedFormatKey</key>
<string>%#@test@</string>
<key>test</key>
<dict>
<key>NSStringFormatSpecTypeKey</key>
<string>NSStringPluralRuleType</string>
<key>NSStringFormatValueTypeKey</key>
<string>d</string>
<key>one</key>
<string>el:One mouse</string>
<key>other</key>
<string>el:%d mice</string>
</dict>
</dict>
</dict>
</plist>
43 changes: 43 additions & 0 deletions openformats/tests/formats/stringsdict/files/1_en.stringsdict
View file
Expand Up @@ -45,5 +45,48 @@
<string>Other tests exist.</string>
</dict>
</dict>
<key>TEST INLINE REPLACEMENT</key>
<dict>
<key>NSStringLocalizedFormatKey</key>
<string>There %#@test@ in the %#@room@ here</string>
<key>test</key>
<dict>
<key>NSStringFormatSpecTypeKey</key>
<string>NSStringPluralRuleType</string>
<key>NSStringFormatValueTypeKey</key>
<string>d</string>
<key>one</key>
<string>is a mouse</string>
<key>other</key>
<string>are %d mice</string>
</dict>
<key>room</key>
<dict>
<key>NSStringFormatSpecTypeKey</key>
<string>NSStringPluralRuleType</string>
<key>NSStringFormatValueTypeKey</key>
<string>d</string>
<key>one</key>
<string>a room</string>
<key>other</key>
<string>%d rooms</string>
</dict>
</dict>
<key>LAST</key>
<dict>
<key>NSStringLocalizedFormatKey</key>
<string>%#@test@</string>
<key>test</key>
<dict>
<key>NSStringFormatSpecTypeKey</key>
<string>NSStringPluralRuleType</string>
<key>NSStringFormatValueTypeKey</key>
<string>d</string>
<key>one</key>
<string>One mouse</string>
<key>other</key>
<string>%d mice</string>
</dict>
</dict>
</dict>
</plist>
37 changes: 37 additions & 0 deletions openformats/tests/formats/stringsdict/files/1_tpl.stringsdict
View file
Expand Up @@ -39,5 +39,42 @@
<tx_awesome_string_tag>ec8a7a8b22906062781b0aacccacfba3_pl</tx_awesome_string_tag>
</dict>
</dict>
<key>TEST INLINE REPLACEMENT</key>
<dict>
<key>NSStringLocalizedFormatKey</key>
<string>%#@test@</string>
<key>test</key>
<dict>
<key>NSStringFormatSpecTypeKey</key>
<string>NSStringPluralRuleType</string>
<key>NSStringFormatValueTypeKey</key>
<string>d</string>
<tx_awesome_key_tag></tx_awesome_key_tag>
<tx_awesome_string_tag>f2173da507415303a5346c293fa086bf_pl</tx_awesome_string_tag>
</dict>
<key>room</key>
<dict>
<key>NSStringFormatSpecTypeKey</key>
<string>NSStringPluralRuleType</string>
<key>NSStringFormatValueTypeKey</key>
<string>d</string>
<tx_awesome_key_tag></tx_awesome_key_tag>
<tx_awesome_string_tag>5941243ff132731d2655e82adf787f14_pl</tx_awesome_string_tag>
</dict>
</dict>
<key>LAST</key>
<dict>
<key>NSStringLocalizedFormatKey</key>
<string>%#@test@</string>
<key>test</key>
<dict>
<key>NSStringFormatSpecTypeKey</key>
<string>NSStringPluralRuleType</string>
<key>NSStringFormatValueTypeKey</key>
<string>d</string>
<tx_awesome_key_tag></tx_awesome_key_tag>
<tx_awesome_string_tag>383613b398803eb778f9abac0e8808f0_pl</tx_awesome_string_tag>
</dict>
</dict>
</dict>
</plist>
12 changes: 12 additions & 0 deletions openformats/tests/formats/stringsdict/test_stringsdict.py
View file
Expand Up @@ -38,6 +38,18 @@ def _create_pluralized_string(self):
context_dict['hash'] = openstring.template_replacement
return context_dict, openstring

def test_compile(self):
"""
Bypass this test.
Stringsdict format differs from the rest of the formats since it
transforms the template in some cases. Compiling a resource using the
source's stringset will not produce the original source file if the
value of any `NSStringLocalizedFormatKey` key contains translatable
content.
"""
pass

def test_string(self):
context_dict, openstring = self._create_pluralized_string()
source = strip_leading_spaces(u"""
Expand Down

0 comments on commit c1c1c4e

Please sign in to comment.