don't index tokens longer than a defined max length (20 for now)

languagetool-org · Nov 27, 2015 · 40a1710 · 40a1710
1 parent 84174ec
commit 40a1710
Showing 1 changed file with 12 additions and 5 deletions.
diff --git a/languagetool-dev/src/main/java/org/languagetool/dev/bigdata/CommonCrawlToNgram.java b/languagetool-dev/src/main/java/org/languagetool/dev/bigdata/CommonCrawlToNgram.java
@@ -51,6 +51,7 @@
 class CommonCrawlToNgram implements AutoCloseable {
 
   private static final double THRESHOLD = 0.00000000001;
+  private static final int MAX_TOKEN_LENGTH = 20;
 
   private final File input;
   private final File indexTopDir;
@@ -128,14 +129,20 @@ private void indexSentence(String sentence) throws IOException {
       if (token.trim().isEmpty()) {
         continue;
       }
-      unigramToCount.compute(token, (k, v) -> v == null ? 1 : v + 1);
+      if (token.length() <= MAX_TOKEN_LENGTH) {
+        unigramToCount.compute(token, (k, v) -> v == null ? 1 : v + 1);
+      }
       if (prev != null) {
-        String ngram = prev + " " + token;
-        bigramToCount.compute(ngram, (k, v) -> v == null ? 1 : v + 1);
+        if (token.length() <= MAX_TOKEN_LENGTH && prev.length() <= MAX_TOKEN_LENGTH) {
+          String ngram = prev + " " + token;
+          bigramToCount.compute(ngram, (k, v) -> v == null ? 1 : v + 1);
+        }
       }
       if (prevPrev != null && prev != null) {
-        String ngram = prevPrev + " " + prev + " " + token;
-        trigramToCount.compute(ngram, (k, v) -> v == null ? 1 : v + 1);
+        if (token.length() <= MAX_TOKEN_LENGTH && prev.length() <= MAX_TOKEN_LENGTH && prevPrev.length() <= MAX_TOKEN_LENGTH) {
+          String ngram = prevPrev + " " + prev + " " + token;
+          trigramToCount.compute(ngram, (k, v) -> v == null ? 1 : v + 1);
+        }
         if (trigramToCount.size() > cacheLimit) {
           writeAndEvaluate();
         }