diff --git a/fluent/migrate/transforms.py b/fluent/migrate/transforms.py index c158f900..cf1f44ca 100644 --- a/fluent/migrate/transforms.py +++ b/fluent/migrate/transforms.py @@ -22,13 +22,13 @@ is equivalent to: FTL.Pattern([ - FTL.TextElement(Source('file.dtd', 'hello')) + Source('file.dtd', 'hello') ]) Sometimes it's useful to work with text rather than (path, key) source -definitions. This is the case when the migrated translation requires some +definitions. This is the case when the migrated translation requires some hardcoded text, e.g. and when multiple translations become a single -one with a DOM overlay. In such cases it's best to use the AST nodes: +one with a DOM overlay. In such cases it's best to use FTL.TextElements: FTL.Message( id=FTL.Identifier('update-failed'), @@ -41,11 +41,10 @@ ) ) -The REPLACE_IN_TEXT Transform also takes text as input, making in possible to -pass it as the foreach function of the PLURALS Transform. In this case, each -slice of the plural string will be run through a REPLACE_IN_TEXT operation. -Those slices are strings, so a REPLACE(path, key, …) wouldn't be suitable for -them. +The REPLACE_IN_TEXT Transform also takes TextElements as input, making it +possible to pass it as the foreach function of the PLURALS Transform. In the +example below, each slice of the plural string is converted into a +TextElement by PLURALS and then run through the REPLACE_IN_TEXT transform. FTL.Message( FTL.Identifier('delete-all'), @@ -64,18 +63,12 @@ """ from __future__ import unicode_literals -import itertools +import re import fluent.syntax.ast as FTL from .errors import NotSupportedError -def pattern_from_text(value): - return FTL.Pattern([ - FTL.TextElement(value) - ]) - - def evaluate(ctx, node): def eval_node(subnode): if isinstance(subnode, Transform): @@ -86,48 +79,113 @@ def eval_node(subnode): return node.traverse(eval_node) +def get_text(element): + '''Get text content of a PatternElement.''' + if isinstance(element, FTL.TextElement): + return element.value + if isinstance(element, FTL.Placeable): + if isinstance(element.expression, FTL.StringExpression): + return element.expression.value + else: + return None + raise RuntimeError('Expected PatternElement') + + +def chain_elements(elements): + '''Flatten a list of FTL nodes into an iterator over PatternElements.''' + for element in elements: + if isinstance(element, FTL.Pattern): + # PY3 yield from element.elements + for child in element.elements: + yield child + elif isinstance(element, FTL.PatternElement): + yield element + elif isinstance(element, FTL.Expression): + yield FTL.Placeable(element) + else: + raise RuntimeError( + 'Expected Pattern, PatternElement or Expression') + + +re_leading_ws = re.compile(r'^(?P\s+)(?P.*?)$') +re_trailing_ws = re.compile(r'^(?P.*?)(?P\s+)$') + + +def extract_whitespace(regex, element): + '''Extract leading or trailing whitespace from a TextElement. + + Return a tuple of (Placeable, TextElement) in which the Placeable + encodes the extracted whitespace as a StringExpression and the + TextElement has the same amount of whitespace removed. The + Placeable with the extracted whitespace is always returned first. + ''' + match = re.search(regex, element.value) + if match: + whitespace = match.group('whitespace') + placeable = FTL.Placeable(FTL.StringExpression(whitespace)) + if whitespace == element.value: + return placeable, None + else: + return placeable, FTL.TextElement(match.group('text')) + else: + return None, element + + class Transform(FTL.BaseNode): def __call__(self, ctx): raise NotImplementedError @staticmethod - def flatten_elements(elements): - '''Flatten a list of FTL nodes into valid Pattern's elements''' - flattened = [] - for element in elements: - if isinstance(element, FTL.Pattern): - flattened.extend(element.elements) - elif isinstance(element, FTL.PatternElement): - flattened.append(element) - elif isinstance(element, FTL.Expression): - flattened.append(FTL.Placeable(element)) + def pattern_of(*elements): + normalized = [] + + # Normalize text content: convert all text to TextElements, join + # adjacent text and prune empty. + for current in chain_elements(elements): + current_text = get_text(current) + if current_text is None: + normalized.append(current) + continue + + previous = normalized[-1] if len(normalized) else None + if isinstance(previous, FTL.TextElement): + # Join adjacent TextElements + previous.value += current_text + elif len(current_text) > 0: + # Normalize non-empty text to a TextElement + normalized.append(FTL.TextElement(current_text)) else: - raise RuntimeError( - 'Expected Pattern, PatternElement or Expression') - return flattened + # Prune empty text + pass - @staticmethod - def prune_text_elements(elements): - '''Join adjacent TextElements and remove empty ones''' - pruned = [] - # Group elements in contiguous sequences of the same type. - for elem_type, elems in itertools.groupby(elements, key=type): - if elem_type is FTL.TextElement: - # Join adjacent TextElements. - text = FTL.TextElement(''.join(elem.value for elem in elems)) - # And remove empty ones. - if len(text.value) > 0: - pruned.append(text) - else: - pruned.extend(elems) - return pruned + # Handle empty values + if len(normalized) == 0: + empty = FTL.Placeable(FTL.StringExpression('')) + return FTL.Pattern([empty]) + + # Handle explicit leading whitespace + if isinstance(normalized[0], FTL.TextElement): + ws, text = extract_whitespace(re_leading_ws, normalized[0]) + normalized[:1] = [ws, text] + + # Handle explicit trailing whitespace + if isinstance(normalized[-1], FTL.TextElement): + ws, text = extract_whitespace(re_trailing_ws, normalized[-1]) + normalized[-1:] = [text, ws] + + return FTL.Pattern([ + element + for element in normalized + if element is not None + ]) class Source(Transform): """Declare the source translation to be migrated with other transforms. - When evaluated, `Source` returns a simple string value. Escaped characters - are unescaped by the compare-locales parser according to the file format: + When evaluated, `Source` returns a TextElement with the content from the + source translation. Escaped characters are unescaped by the + compare-locales parser according to the file format: - in properties files: \\uXXXX, - in DTD files: known named, decimal, and hexadecimal HTML entities. @@ -149,48 +207,49 @@ def __init__(self, path, key): self.key = key def __call__(self, ctx): - return ctx.get_source(self.path, self.key) + text = ctx.get_source(self.path, self.key) + return FTL.TextElement(text) class COPY(Source): """Create a Pattern with the translation value from the given source.""" def __call__(self, ctx): - source = super(self.__class__, self).__call__(ctx) - return pattern_from_text(source) + element = super(self.__class__, self).__call__(ctx) + return Transform.pattern_of(element) class REPLACE_IN_TEXT(Transform): - """Replace various placeables in the translation with FTL. + """Create a Pattern from a TextElement and replace legacy placeables. The original placeables are defined as keys on the `replacements` dict. For each key the value is defined as a FTL Pattern, Placeable, TextElement or Expressions to be interpolated. """ - def __init__(self, value, replacements): - self.value = value + def __init__(self, element, replacements): + self.element = element self.replacements = replacements def __call__(self, ctx): - # Only replace placeables which are present in the translation. replacements = { key: evaluate(ctx, repl) for key, repl in self.replacements.iteritems() - if key in self.value + if key in self.element.value } # Order the original placeables by their position in the translation. keys_in_order = sorted( replacements.keys(), - lambda x, y: self.value.find(x) - self.value.find(y) + lambda x, y: + self.element.value.find(x) - self.element.value.find(y) ) # A list of PatternElements built from the legacy translation and the # FTL replacements. It may contain empty or adjacent TextElements. elements = [] - tail = self.value + tail = self.element.value # Convert original placeables and text into FTL Nodes. For each # original placeable the translation will be partitioned around it and @@ -203,10 +262,7 @@ def __call__(self, ctx): # Dont' forget about the tail after the loop ends. elements.append(FTL.TextElement(tail)) - - elements = self.flatten_elements(elements) - elements = self.prune_text_elements(elements) - return FTL.Pattern(elements) + return Transform.pattern_of(*elements) class REPLACE(Source): @@ -221,59 +277,77 @@ def __init__(self, path, key, replacements): self.replacements = replacements def __call__(self, ctx): - value = super(self.__class__, self).__call__(ctx) - return REPLACE_IN_TEXT(value, self.replacements)(ctx) + element = super(self.__class__, self).__call__(ctx) + return REPLACE_IN_TEXT(element, self.replacements)(ctx) class PLURALS(Source): """Create a Pattern with plurals from given source. Build an `FTL.SelectExpression` with the supplied `selector` and variants - extracted from the source. The source needs to be a semicolon-separated - list of variants. Each variant will be run through the `foreach` function, - which should return an `FTL.Node` or a `Transform`. By default, the - `foreach` function transforms the source text into a Pattern with a single - TextElement. + extracted from the source. The original translation should be a + semicolon-separated list of plural forms. Each form will be converted + into a TextElement and run through the `foreach` function, which should + return an `FTL.Node` or a `Transform`. By default, the `foreach` function + creates a valid Pattern from the TextElement passed into it. """ DEFAULT_ORDER = ('zero', 'one', 'two', 'few', 'many', 'other') - def __init__(self, path, key, selector, foreach=pattern_from_text): + def __init__(self, path, key, selector, foreach=Transform.pattern_of): super(self.__class__, self).__init__(path, key) self.selector = selector self.foreach = foreach def __call__(self, ctx): - value = super(self.__class__, self).__call__(ctx) + element = super(self.__class__, self).__call__(ctx) selector = evaluate(ctx, self.selector) - variants = value.split(';') keys = ctx.plural_categories - - # A special case for languages with one plural category. We don't need - # to insert a SelectExpression at all for them. - if len(keys) == len(variants) == 1: - variant, = variants - return evaluate(ctx, self.foreach(variant)) - - # The default CLDR form should be the last we have in - # DEFAULT_ORDER, usually `other`, but in some cases `many`. - # If we don't have a variant for that, we'll append one, - # using the, in CLDR order, last existing variant in the legacy - # translation. That may or may not be the last variant. + forms = [ + FTL.TextElement(part) + for part in element.value.split(';') + ] + + # The default CLDR form should be the last we have in DEFAULT_ORDER, + # usually `other`, but in some cases `many`. If we don't have a variant + # for that, we'll append one, using the, in CLDR order, last existing + # variant in the legacy translation. That may or may not be the last + # variant. default_key = [ key for key in reversed(self.DEFAULT_ORDER) if key in keys ][0] - keys_and_variants = zip(keys, variants) - keys_and_variants.sort(key=lambda (k, v): self.DEFAULT_ORDER.index(k)) - last_key, last_variant = keys_and_variants[-1] + # Match keys to legacy forms in the order they are defined in Gecko's + # PluralForm.jsm. Filter out empty forms. + pairs = [ + (key, var) + for key, var in zip(keys, forms) + if var.value + ] + + # A special case for legacy translations which don't define any + # plural forms. + if len(pairs) == 0: + return Transform.pattern_of() + + # A special case for languages with one plural category or one legacy + # variant. We don't need to insert a SelectExpression for them. + if len(pairs) == 1: + _, only_form = pairs[0] + only_variant = evaluate(ctx, self.foreach(only_form)) + return Transform.pattern_of(only_variant) + + # Make sure the default key is defined. If it's missing, use the last + # form (in CLDR order) found in the legacy translation. + pairs.sort(key=lambda pair: self.DEFAULT_ORDER.index(pair[0])) + last_key, last_form = pairs[-1] if last_key != default_key: - keys_and_variants.append((default_key, last_variant)) + pairs.append((default_key, last_form)) - def createVariant(key, variant): - # Run the legacy variant through `foreach` which returns an + def createVariant(key, form): + # Run the legacy plural form through `foreach` which returns an # `FTL.Node` describing the transformation required for each - # variant. Then evaluate it to a migrated FTL node. - value = evaluate(ctx, self.foreach(variant)) + # variant. Then evaluate it to a migrated FTL node. + value = evaluate(ctx, self.foreach(form)) return FTL.Variant( key=FTL.VariantName(key), value=value, @@ -283,13 +357,12 @@ def createVariant(key, variant): select = FTL.SelectExpression( expression=selector, variants=[ - createVariant(key, variant) - for key, variant in keys_and_variants + createVariant(key, form) + for key, form in pairs ] ) - placeable = FTL.Placeable(select) - return FTL.Pattern([placeable]) + return Transform.pattern_of(select) class CONCAT(Transform): @@ -303,6 +376,4 @@ def __init__(self, *elements, **kwargs): self.elements = list(kwargs.get('elements', elements)) def __call__(self, ctx): - elements = self.flatten_elements(self.elements) - elements = self.prune_text_elements(elements) - return FTL.Pattern(elements) + return Transform.pattern_of(*self.elements) diff --git a/tests/migrate/test_concat.py b/tests/migrate/test_concat.py index 1dbbb938..6d0ab3d1 100644 --- a/tests/migrate/test_concat.py +++ b/tests/migrate/test_concat.py @@ -25,6 +25,9 @@ def setUp(self): hello = Hello, world! hello.start = Hello,\\u0020 hello.end = world! + empty = + empty.start = + empty.end = whitespace.begin.start = \\u0020Hello,\\u0020 whitespace.begin.end = world! whitespace.end.start = Hello,\\u0020 @@ -55,32 +58,60 @@ def test_concat_two(self): ) ) - result = evaluate(self, msg) + self.assertEqual( + evaluate(self, msg).to_json(), + ftl_message_to_json(''' + hello = Hello, world! + ''') + ) + + def test_concat_empty_one(self): + msg = FTL.Message( + FTL.Identifier('empty'), + value=CONCAT( + COPY('test.properties', 'empty'), + ) + ) self.assertEqual( - len(result.value.elements), - 1, - 'The constructed value should have only one element' + evaluate(self, msg).to_json(), + ftl_message_to_json(''' + empty = {""} + ''') ) - self.assertIsInstance( - result.value.elements[0], - FTL.TextElement, - 'The constructed element should be a TextElement.' + + def test_concat_empty_two(self): + msg = FTL.Message( + FTL.Identifier('empty'), + value=CONCAT( + COPY('test.properties', 'empty.start'), + COPY('test.properties', 'empty.end'), + ) ) + self.assertEqual( - result.value.elements[0].value, - 'Hello, world!', - 'The TextElement should be a concatenation of the sources.' + evaluate(self, msg).to_json(), + ftl_message_to_json(''' + empty = {""} + ''') + ) + + def test_concat_nonempty_empty(self): + msg = FTL.Message( + FTL.Identifier('combined'), + value=CONCAT( + COPY('test.properties', 'hello'), + COPY('test.properties', 'empty'), + ) ) self.assertEqual( - result.to_json(), + evaluate(self, msg).to_json(), ftl_message_to_json(''' - hello = Hello, world! + combined = Hello, world! ''') ) - @unittest.skip('Parser/Serializer trim whitespace') def test_concat_whitespace_begin(self): msg = FTL.Message( FTL.Identifier('hello'), @@ -97,7 +128,6 @@ def test_concat_whitespace_begin(self): ''') ) - @unittest.skip('Parser/Serializer trim whitespace') def test_concat_whitespace_end(self): msg = FTL.Message( FTL.Identifier('hello'), @@ -110,7 +140,7 @@ def test_concat_whitespace_end(self): self.assertEqual( evaluate(self, msg).to_json(), ftl_message_to_json(''' - hello = Hello, world! + hello = Hello, world!{" "} ''') ) diff --git a/tests/migrate/test_context_real_examples.py b/tests/migrate/test_context_real_examples.py index deaf9cd9..3ac83e84 100644 --- a/tests/migrate/test_context_real_examples.py +++ b/tests/migrate/test_context_real_examples.py @@ -321,7 +321,7 @@ def setUp(self): 'aboutDialog.dtd', 'community.mozillaLink', { - '&vendorBrandShortName;': MESSAGE_REFERENCE( + '&vendorShortName;': MESSAGE_REFERENCE( 'vendor-short-name' ) } @@ -336,7 +336,6 @@ def setUp(self): ), ]) - @unittest.skip('Parser/Serializer trim whitespace') def test_merge_context_all_messages(self): expected = { 'aboutDialog.ftl': ftl_resource_to_json(''' @@ -345,8 +344,8 @@ def test_merge_context_all_messages(self): # file, You can obtain one at http://mozilla.org/MPL/2.0/. update-failed = Aktualizacja się nie powiodła. Pobierz. - channel-desc = Obecnie korzystasz z kanału { $channelname }. - community = Program { $brand-short-name } został opracowany przez organizację { $vendor-short-name }, która jest globalną społecznością, starającą się zapewnić, by… + channel-desc = Obecnie korzystasz z kanału { $channelname }.{" "} + community = Program { brand-short-name } został opracowany przez organizację { vendor-short-name }, która jest globalną społecznością, starającą się zapewnić, by… ''') } diff --git a/tests/migrate/test_copy.py b/tests/migrate/test_copy.py index 5def8a39..ad8fbfee 100644 --- a/tests/migrate/test_copy.py +++ b/tests/migrate/test_copy.py @@ -10,6 +10,8 @@ class MockContext(unittest.TestCase): + maxDiff = None + def get_source(self, path, key): # Ignore path (test.properties) and get translations from self.strings # defined in setUp. @@ -20,8 +22,12 @@ class TestCopy(MockContext): def setUp(self): self.strings = parse(PropertiesParser, ''' foo = Foo - foo.unicode.begin = \\u0020Foo - foo.unicode.end = Foo\\u0020 + empty = + unicode.all = \\u0020 + unicode.begin1 = \\u0020Foo + unicode.begin2 = \\u0020\\u0020Foo + unicode.end1 = Foo\\u0020 + unicode.end2 = Foo\\u0020\\u0020 ''') def test_copy(self): @@ -37,31 +43,81 @@ def test_copy(self): ''') ) - @unittest.skip('Parser/Serializer trim whitespace') + def test_copy_empty(self): + msg = FTL.Message( + FTL.Identifier('empty'), + value=COPY('test.properties', 'empty') + ) + + self.assertEqual( + evaluate(self, msg).to_json(), + ftl_message_to_json(''' + empty = {""} + ''') + ) + + def test_copy_escape_unicode_all(self): + msg = FTL.Message( + FTL.Identifier('unicode-all'), + value=COPY('test.properties', 'unicode.all') + ) + + self.assertEqual( + evaluate(self, msg).to_json(), + ftl_message_to_json(''' + unicode-all = {" "} + ''') + ) + def test_copy_escape_unicode_begin(self): msg = FTL.Message( - FTL.Identifier('foo-unicode-begin'), - value=COPY('test.properties', 'foo.unicode.begin') + FTL.Identifier('unicode-begin'), + value=COPY('test.properties', 'unicode.begin1') ) self.assertEqual( evaluate(self, msg).to_json(), ftl_message_to_json(''' - foo-unicode-begin = Foo + unicode-begin = {" "}Foo + ''') + ) + + def test_copy_escape_unicode_begin_many(self): + msg = FTL.Message( + FTL.Identifier('unicode-begin'), + value=COPY('test.properties', 'unicode.begin2') + ) + + self.assertEqual( + evaluate(self, msg).to_json(), + ftl_message_to_json(''' + unicode-begin = {" "}Foo ''') ) - @unittest.skip('Parser/Serializer trim whitespace') def test_copy_escape_unicode_end(self): msg = FTL.Message( - FTL.Identifier('foo-unicode-end'), - value=COPY('test.properties', 'foo.unicode.end') + FTL.Identifier('unicode-end'), + value=COPY('test.properties', 'unicode.end1') + ) + + self.assertEqual( + evaluate(self, msg).to_json(), + ftl_message_to_json(''' + unicode-end = Foo{" "} + ''') + ) + + def test_copy_escape_unicode_end_many(self): + msg = FTL.Message( + FTL.Identifier('unicode-end'), + value=COPY('test.properties', 'unicode.end2') ) self.assertEqual( evaluate(self, msg).to_json(), ftl_message_to_json(''' - foo-unicode-end = Foo + unicode-end = Foo{" "} ''') ) @@ -71,6 +127,7 @@ def setUp(self): self.strings = parse(DTDParser, ''' + ''') def test_copy_accesskey(self): @@ -87,15 +144,22 @@ def test_copy_accesskey(self): 'test.properties', 'checkForUpdatesButton.accesskey' ) ), + FTL.Attribute( + FTL.Identifier('empty'), + COPY( + 'test.properties', 'checkForUpdatesButton.empty' + ) + ), ] ) self.assertEqual( evaluate(self, msg).to_json(), ftl_message_to_json(''' - check-for-updates + check-for-updates = .label = Check for updates .accesskey = C + .empty = {""} ''') ) diff --git a/tests/migrate/test_plural.py b/tests/migrate/test_plural.py index 07a0688d..53ef55b2 100644 --- a/tests/migrate/test_plural.py +++ b/tests/migrate/test_plural.py @@ -12,8 +12,6 @@ class MockContext(unittest.TestCase): maxDiff = None - # Plural categories corresponding to English (en-US). - plural_categories = ('one', 'other') def get_source(self, path, key): # Ignore path (test.properties) and get translations from self.strings @@ -24,26 +22,28 @@ def get_source(self, path, key): class TestPlural(MockContext): def setUp(self): self.strings = parse(PropertiesParser, ''' - deleteAll=Delete this download?;Delete all downloads? + plural = One;Few;Many ''') self.message = FTL.Message( - FTL.Identifier('delete-all'), + FTL.Identifier('plural'), value=PLURALS( 'test.properties', - 'deleteAll', + 'plural', EXTERNAL_ARGUMENT('num') ) ) def test_plural(self): + self.plural_categories = ('one', 'few', 'many') self.assertEqual( evaluate(self, self.message).to_json(), ftl_message_to_json(''' - delete-all = + plural = { $num -> - [one] Delete this download? - *[other] Delete all downloads? + [one] One + [few] Few + *[many] Many } ''') ) @@ -53,43 +53,43 @@ def test_plural_too_few_variants(self): self.assertEqual( evaluate(self, self.message).to_json(), ftl_message_to_json(''' - delete-all = + plural = { $num -> - [one] Delete this download? - [few] Delete all downloads? - *[other] Delete all downloads? + [one] One + [few] Few + [many] Many + *[other] Many } ''') ) def test_plural_too_many_variants(self): - self.plural_categories = ('one',) + self.plural_categories = ('one', 'few') self.assertEqual( evaluate(self, self.message).to_json(), ftl_message_to_json(''' - delete-all = + plural = { $num -> - *[one] Delete this download? + [one] One + *[few] Few } ''') ) class TestPluralOrder(MockContext): - # Plural categories corresponding to Lithuanian (lt). plural_categories = ('one', 'other', 'few') def setUp(self): self.strings = parse(PropertiesParser, ''' - # These forms correspond to (one, other, few) in CLDR - deleteAll = Pašalinti #1 atsiuntimą?;Pašalinti #1 atsiuntimų?;Pašalinti #1 atsiuntimus? + plural = One;Other;Few ''') self.message = FTL.Message( - FTL.Identifier('delete-all'), + FTL.Identifier('plural'), value=PLURALS( 'test.properties', - 'deleteAll', + 'plural', EXTERNAL_ARGUMENT('num') ) ) @@ -98,151 +98,185 @@ def test_unordinary_order(self): self.assertEqual( evaluate(self, self.message).to_json(), ftl_message_to_json(''' - delete-all = + plural = { $num -> - [one] Pašalinti #1 atsiuntimą? - [few] Pašalinti #1 atsiuntimus? - *[other] Pašalinti #1 atsiuntimų? + [one] One + [few] Few + *[other] Other } ''') ) -class TestPluralLiteral(MockContext): +class TestPluralReplace(MockContext): + plural_categories = ('one', 'few', 'many') + def setUp(self): self.strings = parse(PropertiesParser, ''' - deleteAll=Delete this download?;Delete all downloads? + plural = One;Few #1;Many #1 ''') - self.message = FTL.Message( - FTL.Identifier('delete-all'), + def test_plural_replace(self): + msg = FTL.Message( + FTL.Identifier('plural'), value=PLURALS( 'test.properties', - 'deleteAll', - EXTERNAL_ARGUMENT('num') + 'plural', + EXTERNAL_ARGUMENT('num'), + lambda text: REPLACE_IN_TEXT( + text, + { + '#1': EXTERNAL_ARGUMENT('num') + } + ) ) ) - def test_plural_literal(self): self.assertEqual( - evaluate(self, self.message).to_json(), + evaluate(self, msg).to_json(), ftl_message_to_json(''' - delete-all = + plural = { $num -> - [one] Delete this download? - *[other] Delete all downloads? + [one] One + [few] Few { $num } + *[many] Many { $num } } ''') ) -class TestPluralReplace(MockContext): +class TestNoPlural(MockContext): def setUp(self): self.strings = parse(PropertiesParser, ''' - deleteAll=Delete this download?;Delete #1 downloads? + plural-other = Other + plural-one-other = One;Other ''') - def test_plural_replace(self): - msg = FTL.Message( - FTL.Identifier('delete-all'), + def test_one_category_one_variant(self): + self.plural_categories = ('other',) + message = FTL.Message( + FTL.Identifier('plural'), value=PLURALS( 'test.properties', - 'deleteAll', - EXTERNAL_ARGUMENT('num'), - lambda text: REPLACE_IN_TEXT( - text, - { - '#1': EXTERNAL_ARGUMENT('num') - } - ) + 'plural-other', + EXTERNAL_ARGUMENT('num') ) ) self.assertEqual( - evaluate(self, msg).to_json(), + evaluate(self, message).to_json(), ftl_message_to_json(''' - delete-all = - { $num -> - [one] Delete this download? - *[other] Delete { $num } downloads? - } + plural = Other ''') ) + def test_one_category_many_variants(self): + self.plural_categories = ('other',) + message = FTL.Message( + FTL.Identifier('plural'), + value=PLURALS( + 'test.properties', + 'plural-one-other', + EXTERNAL_ARGUMENT('num') + ) + ) -class TestOneCategory(MockContext): - # Plural categories corresponding to Turkish (tr). - plural_categories = ('other',) - - def setUp(self): - self.strings = parse(PropertiesParser, ''' - deleteAll=#1 indirme silinsin mi? - ''') + self.assertEqual( + evaluate(self, message).to_json(), + ftl_message_to_json(''' + plural = One + ''') + ) - self.message = FTL.Message( - FTL.Identifier('delete-all'), + def test_many_categories_one_variant(self): + self.plural_categories = ('one', 'other') + message = FTL.Message( + FTL.Identifier('plural'), value=PLURALS( 'test.properties', - 'deleteAll', - EXTERNAL_ARGUMENT('num'), - lambda text: REPLACE_IN_TEXT( - text, - { - '#1': EXTERNAL_ARGUMENT('num') - } - ) + 'plural-other', + EXTERNAL_ARGUMENT('num') ) ) - def test_no_select_expression(self): self.assertEqual( - evaluate(self, self.message).to_json(), + evaluate(self, message).to_json(), ftl_message_to_json(''' - delete-all = { $num } indirme silinsin mi? + plural = Other ''') ) -class TestManyCategories(MockContext): - # Plural categories corresponding to Polish (pl). +class TestEmpty(MockContext): plural_categories = ('one', 'few', 'many') def setUp(self): - self.strings = parse(PropertiesParser, ''' - deleteAll=Usunąć plik?;Usunąć #1 pliki? - ''') - self.message = FTL.Message( - FTL.Identifier('delete-all'), + FTL.Identifier('plural'), value=PLURALS( 'test.properties', - 'deleteAll', - EXTERNAL_ARGUMENT('num'), - lambda text: REPLACE_IN_TEXT( - text, - { - '#1': EXTERNAL_ARGUMENT('num') - } - ) + 'plural', + EXTERNAL_ARGUMENT('num') ) ) - def test_too_few_variants(self): - # StringBundle's plural rule #9 used for Polish has three categories - # which is one fewer than the CLDR's. The migrated string will not have - # the [other] variant and [many] will be marked as the default. + def test_non_default_empty(self): + self.strings = parse(PropertiesParser, ''' + plural = ;Few;Many + ''') + + self.assertEqual( + evaluate(self, self.message).to_json(), + ftl_message_to_json(''' + plural = + { $num -> + [few] Few + *[many] Many + } + ''') + ) + + def test_default_empty(self): + self.strings = parse(PropertiesParser, ''' + plural = One;Few; + ''') + self.assertEqual( evaluate(self, self.message).to_json(), ftl_message_to_json(''' - delete-all = + plural = { $num -> - [one] Usunąć plik? - [few] Usunąć { $num } pliki? - *[many] Usunąć { $num } pliki? + [one] One + [few] Few + *[many] Few } ''') ) + def test_all_empty(self): + self.strings = parse(PropertiesParser, ''' + plural = ; + ''') + + self.assertEqual( + evaluate(self, self.message).to_json(), + ftl_message_to_json(''' + plural = {""} + ''') + ) + + def test_no_value(self): + self.strings = parse(PropertiesParser, ''' + plural = + ''') + + self.assertEqual( + evaluate(self, self.message).to_json(), + ftl_message_to_json(''' + plural = {""} + ''') + ) + if __name__ == '__main__': unittest.main() diff --git a/tests/migrate/test_replace.py b/tests/migrate/test_replace.py index b92b9330..18449f4f 100644 --- a/tests/migrate/test_replace.py +++ b/tests/migrate/test_replace.py @@ -22,12 +22,32 @@ def get_source(self, path, key): class TestReplace(MockContext): def setUp(self): self.strings = parse(PropertiesParser, ''' + empty = hello = Hello, #1! welcome = Welcome, #1, to #2! first = #1 Bar last = Foo #1 ''') + def test_replace_empty(self): + msg = FTL.Message( + FTL.Identifier(u'empty'), + value=REPLACE( + 'test.properties', + 'empty', + { + '#1': EXTERNAL_ARGUMENT('arg') + } + ) + ) + + self.assertEqual( + evaluate(self, msg).to_json(), + ftl_message_to_json(''' + empty = {""} + ''') + ) + def test_replace_one(self): msg = FTL.Message( FTL.Identifier(u'hello'), diff --git a/tests/migrate/test_source.py b/tests/migrate/test_source.py index 3d2580a6..7f124d2a 100644 --- a/tests/migrate/test_source.py +++ b/tests/migrate/test_source.py @@ -62,7 +62,10 @@ class TestProperties(MockContext): def setUp(self): self.strings = parse(PropertiesParser, ''' foo = Foo + value-empty = + value-whitespace = + unicode-all = \\u0020 unicode-start = \\u0020Foo unicode-middle = Foo\\u0020Bar unicode-end = Foo\\u0020 @@ -72,23 +75,51 @@ def setUp(self): def test_simple_text(self): source = Source('test.properties', 'foo') - self.assertEqual(source(self), 'Foo') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, 'Foo') + + def test_empty_value(self): + source = Source('test.properties', 'value-empty') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, '') + + def test_whitespace_value(self): + source = Source('test.properties', 'value-whitespace') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, '') + + def test_escape_unicode_all(self): + source = Source('test.properties', 'unicode-all') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, ' ') def test_escape_unicode_start(self): source = Source('test.properties', 'unicode-start') - self.assertEqual(source(self), ' Foo') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, ' Foo') def test_escape_unicode_middle(self): source = Source('test.properties', 'unicode-middle') - self.assertEqual(source(self), 'Foo Bar') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, 'Foo Bar') def test_escape_unicode_end(self): source = Source('test.properties', 'unicode-end') - self.assertEqual(source(self), 'Foo ') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, 'Foo ') def test_html_entity(self): source = Source('test.properties', 'html-entity') - self.assertEqual(source(self), '<⇧⌘K>') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, '<⇧⌘K>') class TestDTD(MockContext): @@ -96,6 +127,9 @@ def setUp(self): self.strings = parse(DTDParser, ''' + + + @@ -107,28 +141,54 @@ def setUp(self): def test_simple_text(self): source = Source('test.dtd', 'foo') - self.assertEqual(source(self), 'Foo') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, 'Foo') + + def test_empty_value(self): + source = Source('test.dtd', 'valueEmpty') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, '') + + def test_whitespace_value(self): + source = Source('test.dtd', 'valueWhitespace') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, ' ') def test_backslash_unicode_escape(self): source = Source('test.dtd', 'unicodeEscape') - self.assertEqual(source(self), 'Foo\\u0020Bar') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, 'Foo\\u0020Bar') def test_named_entity(self): source = Source('test.dtd', 'named') - self.assertEqual(source(self), '&') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, '&') def test_decimal_entity(self): source = Source('test.dtd', 'decimal') - self.assertEqual(source(self), '&') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, '&') def test_shorthex_entity(self): source = Source('test.dtd', 'shorthexcode') - self.assertEqual(source(self), '&') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, '&') def test_longhex_entity(self): source = Source('test.dtd', 'longhexcode') - self.assertEqual(source(self), '&') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, '&') def test_unknown_entity(self): source = Source('test.dtd', 'unknown') - self.assertEqual(source(self), '&unknownEntity;') + element = source(self) + self.assertIsInstance(element, FTL.TextElement) + self.assertEqual(element.value, '&unknownEntity;')