apache#9 - Adding "required" param for the ValueFeatures as Michael s…

…uggestion

solr/contrib/ltr/README.md

@@ -159,7 +159,12 @@ using standard Solr queries. As an example:
   "name" : "userTextTitleMatch",
   "type" : "org.apache.solr.ltr.feature.impl.SolrFeature",
   "params" : { "q" : "{!field f=title}${user_text}" }
-}
+},
+ {
+   "name" : "userFromMobile",
+   "type" : "org.apache.solr.ltr.feature.impl.ValueFeature",
+   "params" : { "value" : ${userFromMobile}, "required":true }
+ }
 ]
 ```
 
@@ -194,6 +199,11 @@ called 'user_text' passed in through the request, and will fire if there is
 a term match for the document field 'title' from the value of the external
 field 'user_text'.  You can provide default values for external features as
 well by specifying ${myField:myDefault}, similar to how you would in a Solr config.
+In this case, the fifth feature (userFromMobile) will be looking for an external parameter
+called 'userFromMobile' passed in through the request, if the ValueFeature is :
+required=true, it will throw an exception if the external feature is not passed
+required=false, it will silently ignore the feature and avoid the scoring ( at Document scoring time, the model will consider 0 as feature value)
+The advantage in defining a feature as not required, where possible, is to avoid wasting caching space and time in calculating the featureScore.
 See the [Run a Rerank Query](#run-a-rerank-query) section for how to pass in external information.
 
 ### Custom Features

solr/contrib/ltr/src/java/org/apache/solr/ltr/feature/impl/ValueFeature.java

@@ -36,6 +36,7 @@ public class ValueFeature extends Feature {
 
   protected float configValue = -1f;
   protected String configValueStr = null;
+  protected boolean required = false;
   /** name of the attribute containing the value of this feature **/
   private static final String VALUE_FIELD = "value";
 
@@ -45,6 +46,9 @@ public ValueFeature() {}
   public void init(String name, NamedParams params, int id)
       throws FeatureException {
     super.init(name, params, id);
+    final Object paramRequired = params.get("required");
+    if (paramRequired != null)
+      this.required = (boolean) paramRequired;
     final Object paramValue = params.get(VALUE_FIELD);
     if (paramValue == null) {
       throw new FeatureException("Missing the field 'value' in params for "
@@ -83,7 +87,9 @@ public ValueFeatureWeight(IndexSearcher searcher, String name,
         final String expandedValue = macroExpander.expand(configValueStr);
         if (expandedValue != null) {
           featureValue = Float.parseFloat(expandedValue);
-        }else{
+        } else if (required) {
+          throw new FeatureException(this.getClass().getCanonicalName() + " requires efi parameter that was not passed in request.");
+        } else {
           featureValue=null;
         }
 

solr/contrib/ltr/src/test-files/featureExamples/external_features.json

@@ -16,6 +16,32 @@
     "type":"org.apache.solr.ltr.feature.impl.OriginalScoreFeature",
     "store": "fstore2",
     "params":{}
+}, {
+    "name" : "occurrences",
+    "type" : "org.apache.solr.ltr.feature.impl.ValueFeature",
+    "store": "fstore3",
+    "params" : {
+        "value" : "${myOcc}",
+        "required" : false
+    }
+}, {
+    "name":"originalScore",
+    "type":"org.apache.solr.ltr.feature.impl.OriginalScoreFeature",
+    "store": "fstore3",
+    "params":{}
+}, {
+    "name" : "popularity",
+    "type" : "org.apache.solr.ltr.feature.impl.ValueFeature",
+    "store": "fstore4",
+    "params" : {
+        "value" : "${myPop}",
+        "required" : true
+    }
+}, {
+    "name":"originalScore",
+    "type":"org.apache.solr.ltr.feature.impl.OriginalScoreFeature",
+    "store": "fstore4",
+    "params":{}
 }, {
     "name" : "titlePhraseMatch",
     "type" : "org.apache.solr.ltr.feature.impl.SolrFeature",

solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/impl/TestExternalFeatures.java

@@ -113,10 +113,6 @@ public void testEfiFeatureExtraction() throws Exception {
     query.add("fl", "[fv]");
     assertJQ("/query" + query.toQueryString(), "/error/msg=='Exception from createWeight for Feature [name=matchedTitle, type=SolrFeature, id=0, params={q={!terms f=title}${user_query}}] org.apache.solr.ltr.feature.impl.SolrFeature.SolrFeatureWeight requires efi parameter that was not passed in request.'");
 
-    // Using nondefault store should still result in error with no efi
-    query.remove("fl");
-    query.add("fl", "fvalias:[fv store=fstore2]");
-    assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='originalScore:0.0'");
     // Adding efi in features section should make it work
     query.remove("fl");
     query.add("fl", "score,fvalias:[fv store=fstore2 efi.myconf=2.3]");
@@ -128,4 +124,40 @@ public void testEfiFeatureExtraction() throws Exception {
     query.add("rq", "{!ltr reRankDocs=3 model=externalmodel efi.user_query=w3}");
     assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='confidence:2.3;originalScore:1.0'");
   }
+
+  @Test
+  public void featureExtraction_valueFeatureImplicitlyNotRequired_shouldNotScoreFeature() throws Exception {
+    final SolrQuery query = new SolrQuery();
+    query.setQuery("*:*");
+    query.add("rows", "1");
+
+    // Efi is explicitly not required, so we do not score the feature
+    query.remove("fl");
+    query.add("fl", "fvalias:[fv store=fstore2]");
+    assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='originalScore:0.0'");
+  }
+
+  @Test
+  public void featureExtraction_valueFeatureExplicitlyNotRequired_shouldNotScoreFeature() throws Exception {
+    final SolrQuery query = new SolrQuery();
+    query.setQuery("*:*");
+    query.add("rows", "1");
+
+    // Efi is explicitly not required, so we do not score the feature
+    query.remove("fl");
+    query.add("fl", "fvalias:[fv store=fstore3]");
+    assertJQ("/query" + query.toQueryString(), "/response/docs/[0]/fvalias=='originalScore:0.0'");
+  }
+
+  @Test
+  public void featureExtraction_valueFeatureRequired_shouldThrowException() throws Exception {
+    final SolrQuery query = new SolrQuery();
+    query.setQuery("*:*");
+    query.add("rows", "1");
+
+    // Using nondefault store should still result in error with no efi when it is required (myPop)
+    query.remove("fl");
+    query.add("fl", "fvalias:[fv store=fstore4]");
+    assertJQ("/query" + query.toQueryString(), "/error/msg=='Exception from createWeight for Feature [name=popularity, type=ValueFeature, id=0, params={value=${myPop}, required=true}] org.apache.solr.ltr.feature.impl.ValueFeature.ValueFeatureWeight requires efi parameter that was not passed in request.'");
+  }
 }

solr/contrib/ltr/src/test/org/apache/solr/ltr/feature/impl/TestValueFeature.java

@@ -106,9 +106,9 @@ public void testValueFeature5() throws Exception {
   }
 
   @Test
-  public void testValueFeature6() throws Exception {
+  public void testValueFeature6_implicitlyNotRequired_shouldReturnOkStatusCode() throws Exception {
     loadFeature("c5", ValueFeature.class.getCanonicalName(), "c5",
-        "{\"value\":\"${val5}\"}");
+        "{\"value\":\"${val6}\"}");
     loadModel("m5", RankSVMModel.class.getCanonicalName(), new String[] {"c5"},
         "c5", "{\"weights\":{\"c5\":1.0}}");
 
@@ -119,13 +119,43 @@ public void testValueFeature6() throws Exception {
     query.add("wt", "json");
     query.add("rq", "{!ltr model=m5 reRankDocs=4}");
 
-    // String res = restTestHarness.query("/query" + query.toQueryString());
-    // System.out.println(res);
+    assertJQ("/query" + query.toQueryString(), "/responseHeader/status==0");
+  }
+
+  @Test
+  public void testValueFeature6_explictlyNotRequired_shouldReturnOkStatusCode() throws Exception {
+    loadFeature("c7", ValueFeature.class.getCanonicalName(), "c7",
+        "{\"value\":\"${val7}\",\"required\":false}");
+    loadModel("m7", RankSVMModel.class.getCanonicalName(), new String[] {"c7"},
+        "c7", "{\"weights\":{\"c7\":1.0}}");
+
+    final SolrQuery query = new SolrQuery();
+    query.setQuery("title:w1");
+    query.add("fl", "*, score,fvonly:[fvonly]");
+    query.add("rows", "4");
+    query.add("wt", "json");
+    query.add("rq", "{!ltr model=m7 reRankDocs=4}");
 
-    // No efi.val passed in
     assertJQ("/query" + query.toQueryString(), "/responseHeader/status==0");
   }
 
+  @Test
+  public void testValueFeature6_required_shouldReturn400StatusCode() throws Exception {
+    loadFeature("c8", ValueFeature.class.getCanonicalName(), "c8",
+        "{\"value\":\"${val8}\",\"required\":true}");
+    loadModel("m8", RankSVMModel.class.getCanonicalName(), new String[] {"c8"},
+        "c8", "{\"weights\":{\"c8\":1.0}}");
+
+    final SolrQuery query = new SolrQuery();
+    query.setQuery("title:w1");
+    query.add("fl", "*, score,fvonly:[fvonly]");
+    query.add("rows", "4");
+    query.add("wt", "json");
+    query.add("rq", "{!ltr model=m8 reRankDocs=4}");
+
+    assertJQ("/query" + query.toQueryString(), "/responseHeader/status==400");
+  }
+
   @Test
   public void testValueFeature7() throws Exception {
     loadFeature("c6", ValueFeature.class.getCanonicalName(), "c6",