From 1368570a8d0c1460d0b8285ca287d27ba74d8f59 Mon Sep 17 00:00:00 2001 From: mnilsson23 Date: Fri, 10 Feb 2017 15:01:29 -0500 Subject: [PATCH] SOLR-10072: Fix test reliability There is no guarantee to order of docs with same score --- .../featureExamples/external_features.json | 6 +- .../modelExamples/external_model.json | 4 +- ..._model_store.json => external_model2.json} | 6 +- .../solr/ltr/TestParallelWeightCreation.java | 24 ++---- .../solr/ltr/TestSelectiveWeightCreation.java | 83 ++++++++++--------- .../ltr/feature/TestExternalFeatures.java | 32 +++---- 6 files changed, 76 insertions(+), 79 deletions(-) rename solr/contrib/ltr/src/test-files/modelExamples/{external_model_store.json => external_model2.json} (61%) diff --git a/solr/contrib/ltr/src/test-files/featureExamples/external_features.json b/solr/contrib/ltr/src/test-files/featureExamples/external_features.json index 6c0cfa634526..d8a9ecae6f95 100644 --- a/solr/contrib/ltr/src/test-files/featureExamples/external_features.json +++ b/solr/contrib/ltr/src/test-files/featureExamples/external_features.json @@ -5,11 +5,11 @@ "q" : "{!terms f=title}${user_query}" } }, { - "name" : "confidence", - "class" : "org.apache.solr.ltr.feature.ValueFeature", + "name" : "docPopularity", + "class" : "org.apache.solr.ltr.feature.FieldValueFeature", "store": "fstore2", "params" : { - "value" : "${myconf}" + "field" : "popularity" } }, { "name":"originalScore", diff --git a/solr/contrib/ltr/src/test-files/modelExamples/external_model.json b/solr/contrib/ltr/src/test-files/modelExamples/external_model.json index 04ab22968b09..68f659c39622 100644 --- a/solr/contrib/ltr/src/test-files/modelExamples/external_model.json +++ b/solr/contrib/ltr/src/test-files/modelExamples/external_model.json @@ -2,11 +2,11 @@ "class":"org.apache.solr.ltr.model.LinearModel", "name":"externalmodel", "features":[ - { "name": "matchedTitle"} + { "name": "titlePhraseMatch"} ], "params":{ "weights": { - "matchedTitle": 0.999 + "titlePhraseMatch": 0.555 } } } diff --git a/solr/contrib/ltr/src/test-files/modelExamples/external_model_store.json b/solr/contrib/ltr/src/test-files/modelExamples/external_model2.json similarity index 61% rename from solr/contrib/ltr/src/test-files/modelExamples/external_model_store.json rename to solr/contrib/ltr/src/test-files/modelExamples/external_model2.json index f8e664855d3a..f3d33f7faeee 100644 --- a/solr/contrib/ltr/src/test-files/modelExamples/external_model_store.json +++ b/solr/contrib/ltr/src/test-files/modelExamples/external_model2.json @@ -1,13 +1,13 @@ { "class":"org.apache.solr.ltr.model.LinearModel", - "name":"externalmodelstore", + "name":"externalmodel2", "store": "fstore2", "features":[ - { "name": "confidence"} + { "name": "docPopularity"} ], "params":{ "weights": { - "confidence": 0.999 + "docPopularity": 0.5 } } } diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java index f4c21fd66073..630a68cf87f9 100644 --- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java +++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestParallelWeightCreation.java @@ -25,21 +25,15 @@ public class TestParallelWeightCreation extends TestRerankBase{ public void testLTRScoringQueryParallelWeightCreationResultOrder() throws Exception { setuptest("solrconfig-ltr_Th10_10.xml", "schema.xml"); - assertU(adoc("id", "1", "title", "w1 w3", "description", "w1", "popularity", - "1")); - assertU(adoc("id", "2", "title", "w2", "description", "w2", "popularity", - "2")); - assertU(adoc("id", "3", "title", "w3", "description", "w3", "popularity", - "3")); - assertU(adoc("id", "4", "title", "w4 w3", "description", "w4", "popularity", - "4")); - assertU(adoc("id", "5", "title", "w5", "description", "w5", "popularity", - "5")); + assertU(adoc("id", "1", "title", "w1 w3", "description", "w1", "popularity", "1")); + assertU(adoc("id", "2", "title", "w2", "description", "w2", "popularity", "2")); + assertU(adoc("id", "3", "title", "w3", "description", "w3", "popularity", "3")); + assertU(adoc("id", "4", "title", "w3 w3", "description", "w4", "popularity", "4")); + assertU(adoc("id", "5", "title", "w5", "description", "w5", "popularity", "5")); assertU(commit()); loadFeatures("external_features.json"); loadModels("external_model.json"); - loadModels("external_model_store.json"); // check to make sure that the order of results will be the same when using parallel weight creation final SolrQuery query = new SolrQuery(); @@ -47,10 +41,10 @@ public void testLTRScoringQueryParallelWeightCreationResultOrder() throws Except query.add("fl", "*,score"); query.add("rows", "4"); - query.add("rq", "{!ltr reRankDocs=4 model=externalmodel efi.user_query=w3}"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='3'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='4'"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'"); aftertest(); } diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java index e44d4ac54966..5cfd999b360c 100644 --- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java +++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestSelectiveWeightCreation.java @@ -106,21 +106,16 @@ private LTRScoringQuery.ModelWeight performQuery(TopDocs hits, public static void before() throws Exception { setuptest(false); - assertU(adoc("id", "1", "title", "w1 w3", "description", "w1", "popularity", - "1")); - assertU(adoc("id", "2", "title", "w2", "description", "w2", "popularity", - "2")); - assertU(adoc("id", "3", "title", "w3", "description", "w3", "popularity", - "3")); - assertU(adoc("id", "4", "title", "w4 w3", "description", "w4", "popularity", - "4")); - assertU(adoc("id", "5", "title", "w5", "description", "w5", "popularity", - "5")); + assertU(adoc("id", "1", "title", "w3 w1", "description", "w1", "popularity", "1")); + assertU(adoc("id", "2", "title", "w2", "description", "w2", "popularity", "2")); + assertU(adoc("id", "3", "title", "w3", "description", "w3", "popularity", "3")); + assertU(adoc("id", "4", "title", "w3 w3", "description", "w4", "popularity", "4")); + assertU(adoc("id", "5", "title", "w5", "description", "w5", "popularity", "5")); assertU(commit()); loadFeatures("external_features.json"); loadModels("external_model.json"); - loadModels("external_model_store.json"); + loadModels("external_model2.json"); } @AfterClass @@ -134,14 +129,14 @@ public void testScoringQueryWeightCreation() throws IOException, ModelException final RandomIndexWriter w = new RandomIndexWriter(random(), dir); Document doc = new Document(); - doc.add(newStringField("id", "0", Field.Store.YES)); + doc.add(newStringField("id", "10", Field.Store.YES)); doc.add(newTextField("field", "wizard the the the the the oz", Field.Store.NO)); doc.add(new FloatDocValuesField("final-score", 1.0f)); w.addDocument(doc); doc = new Document(); - doc.add(newStringField("id", "1", Field.Store.YES)); + doc.add(newStringField("id", "11", Field.Store.YES)); // 1 extra token, but wizard and oz are close; doc.add(newTextField("field", "wizard oz the the the the the the", Field.Store.NO)); @@ -159,8 +154,8 @@ public void testScoringQueryWeightCreation() throws IOException, ModelException // first run the standard query final TopDocs hits = searcher.search(bqBuilder.build(), 10); assertEquals(2, hits.totalHits); - assertEquals("0", searcher.doc(hits.scoreDocs[0].doc).get("id")); - assertEquals("1", searcher.doc(hits.scoreDocs[1].doc).get("id")); + assertEquals("10", searcher.doc(hits.scoreDocs[0].doc).get("id")); + assertEquals("11", searcher.doc(hits.scoreDocs[1].doc).get("id")); List features = makeFeatures(new int[] {0, 1, 2}); final List allFeatures = makeFeatures(new int[] {0, 1, 2, 3, 4, 5, @@ -206,7 +201,7 @@ public void testScoringQueryWeightCreation() throws IOException, ModelException } assertEquals(validFeatures, allFeatures.size()); - assertU(delI("0"));assertU(delI("1")); + assertU(delI("10"));assertU(delI("11")); r.close(); dir.close(); } @@ -216,41 +211,49 @@ public void testScoringQueryWeightCreation() throws IOException, ModelException public void testSelectiveWeightsRequestFeaturesFromDifferentStore() throws Exception { final String docs0fv = FeatureLoggerTestUtils.toFeatureVector( - "matchedTitle","1.0", "titlePhraseMatch","0.40254828"); + "matchedTitle","1.0", "titlePhraseMatch","0.6103343"); final String docs0fv_fstore4= FeatureLoggerTestUtils.toFeatureVector( "popularity","3.0", "originalScore","1.0"); + // extract all features in externalmodel's store (default store) + // rerank using externalmodel (default store) final SolrQuery query = new SolrQuery(); query.setQuery("*:*"); - query.add("fl", "*,score"); - query.add("rows", "4"); - query.add("rq", "{!ltr reRankDocs=4 model=externalmodel efi.user_query=w3}"); - query.add("fl", "fv:[fv]"); - - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='3'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='4'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv+"'"); // extract all features in default store - + query.add("fl", "*,score,fv:[fv]"); + query.add("rows", "5"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}"); + + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv+"'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.33873552"); + + // extract all features from fstore4 + // rerank using externalmodel (default store) query.remove("fl"); query.remove("rq"); - query.add("fl", "*,score"); - query.add("rq", "{!ltr reRankDocs=4 model=externalmodel efi.user_query=w3}"); - query.add("fl", "fv:[fv store=fstore4 efi.myPop=3]"); - - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.999"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'"); // extract all features from fstore4 + query.add("fl", "*,score,fv:[fv store=fstore4 efi.myPop=3]"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='1'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.33873552"); + // extract all features from fstore4 + // rerank using externalmodel2 (fstore2) query.remove("fl"); query.remove("rq"); - query.add("fl", "*,score"); - query.add("rq", "{!ltr reRankDocs=4 model=externalmodelstore efi.user_query=w3 efi.myconf=0.8}"); - query.add("fl", "fv:[fv store=fstore4 efi.myPop=3]"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='1'"); // score using fstore2 used by externalmodelstore - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.7992"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'"); // extract all features from fstore4 + query.add("fl", "*,score,fv:[fv store=fstore4 efi.myPop=3]"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel2 efi.user_query=w3}"); + + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='5'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='4'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='3'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fv=='"+docs0fv_fstore4+"'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==2.5"); } } diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java index 10ababb07f5a..4010ee1900cb 100644 --- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java +++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/TestExternalFeatures.java @@ -65,24 +65,24 @@ public void testEfiInTransformerShouldNotChangeOrderOfRerankedResults() throws E assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='3'"); assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/score==1.0"); - query.add("fl", "[fv]"); - query.add("rq", "{!ltr reRankDocs=3 model=externalmodel efi.user_query=w3}"); + query.remove("fl"); + query.add("fl", "*,score,[fv]"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}"); assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.999"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='1'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.7693934"); assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/score==0.0"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='2'"); assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/score==0.0"); // Adding an efi in the transformer should not affect the rq ranking with a // different value for efi of the same parameter query.remove("fl"); - query.add("fl", "id,[fv efi.user_query=w2]"); + query.add("fl", "*,score,[fv efi.user_query=w2]"); assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/id=='3'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/id=='1'"); - assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/id=='2'"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/score==0.7693934"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[1]/score==0.0"); + assertJQ("/query" + query.toQueryString(), "/response/docs/[2]/score==0.0"); } @Test @@ -92,7 +92,7 @@ public void testFeaturesUseStopwordQueryReturnEmptyFeatureVector() throws Except query.add("fl", "*,score,fv:[fv]"); query.add("rows", "1"); // Stopword only query passed in - query.add("rq", "{!ltr reRankDocs=3 model=externalmodel efi.user_query='a'}"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query='a'}"); final String docs0fv_dense_csv = FeatureLoggerTestUtils.toFeatureVector( "matchedTitle","0.0", @@ -112,7 +112,7 @@ public void testEfiFeatureExtraction() throws Exception { query.add("rows", "1"); final String docs0fv_csv = FeatureLoggerTestUtils.toFeatureVector( - "confidence","2.3", "originalScore","1.0"); + "occurrences","2.3", "originalScore","1.0"); // Features we're extracting depend on external feature info not passed in query.add("fl", "[fv]"); @@ -120,13 +120,13 @@ public void testEfiFeatureExtraction() throws Exception { // Adding efi in features section should make it work query.remove("fl"); - query.add("fl", "score,fvalias:[fv store=fstore2 efi.myconf=2.3]"); + query.add("fl", "score,fvalias:[fv store=fstore3 efi.myOcc=2.3]"); assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='"+docs0fv_csv+"'"); - // Adding efi in transformer + rq should still use the transformer's params for feature extraction + // Adding efi in transformer + rq should still returns features query.remove("fl"); - query.add("fl", "score,fvalias:[fv store=fstore2 efi.myconf=2.3]"); - query.add("rq", "{!ltr reRankDocs=3 model=externalmodel efi.user_query=w3}"); + query.add("fl", "score,fvalias:[fv store=fstore3 efi.myOcc=2.3]"); + query.add("rq", "{!ltr reRankDocs=10 model=externalmodel efi.user_query=w3}"); assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='"+docs0fv_csv+"'"); } @@ -137,7 +137,7 @@ public void featureExtraction_valueFeatureImplicitlyNotRequired_shouldNotScoreFe query.add("rows", "1"); final String docs0fvalias_dense_csv = FeatureLoggerTestUtils.toFeatureVector( - "confidence","0.0", + "occurrences","0.0", "originalScore","0.0"); final String docs0fvalias_sparse_csv = FeatureLoggerTestUtils.toFeatureVector( "originalScore","0.0"); @@ -146,7 +146,7 @@ public void featureExtraction_valueFeatureImplicitlyNotRequired_shouldNotScoreFe // Efi is explicitly not required, so we do not score the feature query.remove("fl"); - query.add("fl", "fvalias:[fv store=fstore2]"); + query.add("fl", "fvalias:[fv store=fstore3]"); assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='"+docs0fvalias_default_csv+"'"); }