diff --git a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilter.java b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilter.java index 0ac65d240b..811958af4c 100644 --- a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilter.java +++ b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilter.java @@ -48,6 +48,9 @@ public class TikaEvalMetadataFilter extends MetadataFilter { public static Property NUM_ALPHA_TOKENS = Property.externalInteger(TIKA_EVAL_NS + "numAlphaTokens"); + public static Property NUM_COMMON_TOKENS = + Property.externalInteger(TIKA_EVAL_NS + "numCommonTokens"); + public static Property NUM_UNIQUE_ALPHA_TOKENS = Property.externalInteger(TIKA_EVAL_NS + "numUniqueAlphaTokens"); @@ -90,6 +93,7 @@ private void calcStats(String content, Metadata metadata) { CommonTokenResult commonTokenResult = (CommonTokenResult) results.get(CommonTokens.class); metadata.set(NUM_ALPHA_TOKENS, commonTokenResult.getAlphabeticTokens()); metadata.set(NUM_UNIQUE_ALPHA_TOKENS, commonTokenResult.getUniqueAlphabeticTokens()); + metadata.set(NUM_COMMON_TOKENS, commonTokenResult.getCommonTokens()); if (commonTokenResult.getAlphabeticTokens() > 0) { metadata.set(OUT_OF_VOCABULARY, commonTokenResult.getOOV()); } else { diff --git a/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilterTest.java b/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilterTest.java index 1961698b4b..f1fd21c21e 100644 --- a/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilterTest.java +++ b/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilterTest.java @@ -42,6 +42,7 @@ public void testBasic() throws Exception { assertEquals(11, (int) metadata.getInt(TikaEvalMetadataFilter.NUM_UNIQUE_TOKENS)); assertEquals(10, (int) metadata.getInt(TikaEvalMetadataFilter.NUM_ALPHA_TOKENS)); assertEquals(9, (int) metadata.getInt(TikaEvalMetadataFilter.NUM_UNIQUE_ALPHA_TOKENS)); + assertEquals(9, (int) metadata.getInt(TikaEvalMetadataFilter.NUM_COMMON_TOKENS)); assertEquals(0.0999,