Respect "NoSubstitution" flag for combining marks applied on spaces (#74

) * Respect "NoSubstitution" flag for combining marks applied on spaces * Fix style
OpenNMT · Jun 12, 2019 · 120c287 · 120c287
1 parent 3025d9e
commit 120c287
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 1 deletion.
diff --git a/src/Tokenizer.cc b/src/Tokenizer.cc
@@ -629,7 +629,7 @@ namespace onmt
                 token.join_left();
               }
 
-              if (sub_c[0] == ' ')
+              if (sub_c[0] == ' ' && !_no_substitution)
                 token.append(protected_character + "0020" + sub_c.substr(1));
               else
                 token.append(sub_c);

diff --git a/test/test.cc b/test/test.cc
@@ -24,6 +24,22 @@ static void test_tok(ITokenizer& tokenizer,
   }
 }
 
+static void test_tok(ITokenizer& tokenizer,
+                     const std::string& in,
+                     const std::vector<std::string>& expected,
+                     bool detokenize = false) {
+  std::vector<std::string> tokens;
+  tokenizer.tokenize(in, tokens);
+  ASSERT_EQ(tokens.size(), expected.size());
+  for (size_t i = 0; i < tokens.size(); ++i) {
+    EXPECT_EQ(tokens[i], expected[i])  << "Unexpected token mismatch at index " << i;
+  }
+  if (detokenize) {
+    auto text = tokenizer.detokenize(tokens);
+    EXPECT_EQ(text, in);
+  }
+}
+
 static void test_detok(ITokenizer& tokenizer, const std::string& in, const std::string& expected) {
   std::vector<std::string> tokens;
   onmt::SpaceTokenizer::get_instance().tokenize(in, tokens);
@@ -262,6 +278,12 @@ TEST(TokenizerTest, MarkOnSpace) {
                      "b ％0020̇ c");
 }
 
+TEST(TokenizerTest, MarkOnSpaceNoSubstitution) {
+  Tokenizer tokenizer(Tokenizer::Mode::Conservative,
+                      Tokenizer::Flags::JoinerAnnotate | Tokenizer::Flags::NoSubstitution);
+  test_tok(tokenizer, "angles ၧ1 and ၧ2", {"angles", "￭ ၧ￭", "1", "and", "￭ ၧ￭", "2"}, true);
+}
+
 TEST(TokenizerTest, CaseFeature) {
   Tokenizer tokenizer(Tokenizer::Mode::Conservative,
                       Tokenizer::Flags::CaseFeature | Tokenizer::Flags::JoinerAnnotate);