TIKA-4193 -- add num common tokens to TikaEvalMetadataFilter (#1582)

apache · Feb 8, 2024 · 16e1bc9 · 16e1bc9
1 parent 255cc79
commit 16e1bc9
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 0 deletions.
diff --git a/...ka-eval-core/src/main/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilter.java b/...ka-eval-core/src/main/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilter.java
@@ -48,6 +48,9 @@ public class TikaEvalMetadataFilter extends MetadataFilter {
     public static Property NUM_ALPHA_TOKENS =
             Property.externalInteger(TIKA_EVAL_NS + "numAlphaTokens");
 
+    public static Property NUM_COMMON_TOKENS =
+            Property.externalInteger(TIKA_EVAL_NS + "numCommonTokens");
+
     public static Property NUM_UNIQUE_ALPHA_TOKENS =
             Property.externalInteger(TIKA_EVAL_NS + "numUniqueAlphaTokens");
 
@@ -90,6 +93,7 @@ private void calcStats(String content, Metadata metadata) {
         CommonTokenResult commonTokenResult = (CommonTokenResult) results.get(CommonTokens.class);
         metadata.set(NUM_ALPHA_TOKENS, commonTokenResult.getAlphabeticTokens());
         metadata.set(NUM_UNIQUE_ALPHA_TOKENS, commonTokenResult.getUniqueAlphabeticTokens());
+        metadata.set(NUM_COMMON_TOKENS, commonTokenResult.getCommonTokens());
         if (commonTokenResult.getAlphabeticTokens() > 0) {
             metadata.set(OUT_OF_VOCABULARY, commonTokenResult.getOOV());
         } else {

diff --git a/...val-core/src/test/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilterTest.java b/...val-core/src/test/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilterTest.java
@@ -42,6 +42,7 @@ public void testBasic() throws Exception {
             assertEquals(11, (int) metadata.getInt(TikaEvalMetadataFilter.NUM_UNIQUE_TOKENS));
             assertEquals(10, (int) metadata.getInt(TikaEvalMetadataFilter.NUM_ALPHA_TOKENS));
             assertEquals(9, (int) metadata.getInt(TikaEvalMetadataFilter.NUM_UNIQUE_ALPHA_TOKENS));
+            assertEquals(9, (int) metadata.getInt(TikaEvalMetadataFilter.NUM_COMMON_TOKENS));
 
 
             assertEquals(0.0999,