[uk] performance optimization for Ukrainian tokenizer

languagetool-org · Jan 22, 2015 · aa134d6 · aa134d6
1 parent 924a883
commit aa134d6
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/...guage-modules/uk/src/main/java/org/languagetool/tokenizers/uk/UkrainianWordTokenizer.java b/...guage-modules/uk/src/main/java/org/languagetool/tokenizers/uk/UkrainianWordTokenizer.java
@@ -70,7 +70,11 @@ public List<String> tokenize(String text) {
   }
 
   private static String cleanup(String text) {
-    return text.replace("\u0301", "").replace("\u00AD", "").replace('’', '\'').replace('ʼ', '\'');
+    text = text.replace('’', '\'').replace('ʼ', '\'');
+    if( text.contains("\u0301") || text.contains("\u00AD") ) {
+      text = text.replace("\u0301", "").replace("\u00AD", "");
+    }
+    return text;
   }
-  
+
 }