Merge pull request #31140 from vespa-engine/marius/add-more-significa…

…nce-searcher-tests

Add additional significance searcher tests

container-search/src/test/java/com/yahoo/search/significance/model/docv1.json

@@ -0,0 +1,18 @@
+{
+  "version" : "1.0",
+  "id" : "test::1",
+  "description" : "desc",
+  "languages" : {
+    "en": {
+      "description" : "english model",
+      "document-count" : 12,
+      "language" : "en",
+      "document-frequencies" : {
+        "usa" : 2,
+        "hello": 3,
+        "world": 5,
+        "test": 2
+      }
+    }
+  }
+}

container-search/src/test/java/com/yahoo/search/significance/model/docv2.json

@@ -0,0 +1,31 @@
+{
+  "version" : "2.0",
+  "id" : "test::2",
+  "description" : "desc",
+  "languages" : {
+    "en": {
+      "description" : "english model",
+      "document-count" : 16,
+      "document-frequencies" : {
+        "usa" : 2,
+        "hello": 3,
+        "world": 5,
+        "test": 4,
+        "additional": 2
+      }
+    },
+    "nb": {
+      "description" : "norwegian model",
+      "document-count" : 20,
+      "document-frequencies" : {
+        "usa" : 2,
+        "hello": 10,
+        "verden": 5,
+        "test": 2,
+        "norge": 11,
+        "ja": 12,
+        "nei": 3
+      }
+    }
+  }
+}

container-search/src/test/java/com/yahoo/search/significance/model/en.json

@@ -5,13 +5,14 @@
   "languages" : {
     "en": {
       "description" : "english model",
-      "document-count" : 10,
+      "document-count" : 16,
       "language" : "en",
       "document-frequencies" : {
         "usa" : 2,
         "hello": 3,
         "world": 5,
-        "test": 2
+        "test": 2,
+        "additional": 2
       }
     }
   }

container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java

@@ -40,8 +40,8 @@ public class SignificanceSearcherTest {
 
     public SignificanceSearcherTest() {
         List<Path> models = new ArrayList<>();
-        models.add( Path.of("src/test/java/com/yahoo/search/significance/model/en.json"));
-
+        models.add(Path.of("src/test/java/com/yahoo/search/significance/model/docv1.json"));
+        models.add(Path.of("src/test/java/com/yahoo/search/significance/model/docv2.json"));
         var schema = new Schema.Builder("music")
                 .add(new DocumentSummary.Builder("default").build())
                 .add(new RankProfile.Builder("significance-ranking")
@@ -61,6 +61,26 @@ private Execution createExecution() {
 
     @Test
     void testSignificanceValueOnSimpleQuery() {
+        Query q = new Query();
+        q.getRanking().setProfile("significance-ranking");
+        AndItem root = new AndItem();
+        WordItem tmp;
+        tmp = new WordItem("hello", true);
+        root.addItem(tmp);
+
+        q.getModel().getQueryTree().setRoot(root);
+
+        SignificanceModel model = significanceModelRegistry.getModel(Language.ENGLISH).get();
+        var helloFrequency = model.documentFrequency("hello");
+        var helloSignificanceValue = SignificanceSearcher.calculateIDF(helloFrequency.corpusSize(), helloFrequency.frequency());
+        Result r = createExecution(searcher).search(q);
+
+        root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot();
+        WordItem w0 = (WordItem) root.getItem(0);
+        assertEquals(helloSignificanceValue, w0.getSignificance());
+    }
+    @Test
+    void testSignificanceValueOnSimpleANDQuery() {
 
         Query q = new Query();
         q.getRanking().setProfile("significance-ranking");
@@ -133,7 +153,7 @@ void testSignificanceValueOnRecursiveQuery() {
 
         assertEquals(helloSignificanceValue, w0.getSignificance());
         assertEquals(testSignificanceValue, w1.getSignificance());
-        assertEquals(SignificanceSearcher.calculateIDF(10, 2), w3.getSignificance());
+        assertEquals(SignificanceSearcher.calculateIDF(16, 2), w3.getSignificance());
 
     }