Merge branch 'master' into clpwsim-lsh

mt3 · Jan 29, 2012 · b1a592c · b1a592c
2 parents 9e14af5 + f156b0c
commit b1a592c
Show file tree

Hide file tree

Showing 15 changed files with 496 additions and 139 deletions.
diff --git a/404.html b/404.html
diff --git a/src/java/main/ivory/ltr/ExtractFeatures.java b/src/java/main/ivory/ltr/ExtractFeatures.java
@@ -44,6 +44,8 @@
 
 import javax.xml.parsers.ParserConfigurationException;
 
+import com.google.common.collect.Maps;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.w3c.dom.Node;
@@ -52,9 +54,14 @@
 
 import edu.umd.cloud9.collection.DocnoMapping;
 
+import ivory.ltr.operator.Operator;
+import ivory.ltr.operator.OperatorUtil;
+import ivory.ltr.operator.Sum;
+
 /**
  * @author Don Metzler
  *
+ * Modified by Nima Asadi
  */
 public class ExtractFeatures {
 
@@ -69,12 +76,24 @@ public class ExtractFeatures {
   private Map<String, String> queries = null; // query id -> query text mapping
   private DocnoMapping docnoMapping = null;   // docno mapping
 
+  private Map<String, Operator> operators = null;
+
   public ExtractFeatures(String [] args, FileSystem fs) throws SAXException, IOException, ParserConfigurationException, NotBoundException, Exception {
     loadQueryRunner(args, fs);
     env = runner.getRetrievalEnvironment();
     queries = runner.getQueries();
     docnoMapping = env.getDocnoMapping();
 
+    operators = Maps.newHashMap();
+    for(String configPath: args) {
+      Map<String, Operator> ops = OperatorUtil.parseOperators(configPath);
+      if(ops.size() > 0) {
+        for(String key: ops.keySet()) {
+          operators.put(key, ops.get(key));
+        }
+      }
+    }
+
     Map<String, String> finalQueries = new HashMap<String, String>();
     for(Entry<String, String> queryEntry : queries.entrySet()) {
       String queryKey = queryEntry.getKey();
@@ -169,7 +188,7 @@ private void extract() throws Exception {
    // extract features query-by-query
    for(Entry<String, String> queryEntry : queries.entrySet()) {
      // feature map (docname -> feature name -> feature value)
-     SortedMap<String,SortedMap<String,Double>> featureValues = new TreeMap<String,SortedMap<String,Double>>();
+     SortedMap<String,SortedMap<String,Operator>> featureValues = new TreeMap<String,SortedMap<String,Operator>>();
 
      // query id and text
      String qid = queryEntry.getKey();
@@ -231,17 +250,18 @@ private void extract() throws Exception {
          String docName = docIdToNameMap.get(docid);
 
          // get feature map for this docname
-         SortedMap<String,Double> docFeatures = featureValues.get(docName);
+         SortedMap<String,Operator> docFeatures = featureValues.get(docName);
          if(docFeatures == null) {
-           docFeatures = new TreeMap<String,Double>();
+           docFeatures = new TreeMap<String,Operator>();
            featureValues.put(docName, docFeatures);
          }
 
          // document judgment
          double judgment = judgmentEntry.getValue();
 
          // set judgment feature
-         docFeatures.put(JUDGMENT_FEATURE_NAME, judgment);
+         docFeatures.put(JUDGMENT_FEATURE_NAME, new Sum());
+         docFeatures.get(JUDGMENT_FEATURE_NAME).addScore(judgment);
 
          // initialize doc nodes
          for(DocumentNode node : docNodes) {
@@ -266,13 +286,10 @@ private void extract() throws Exception {
                  double score = model.computeFeatureValue(c.getConcept(), metaFeat) * c.getPotential();
 
                  // update feature values
-                 Double curVal = docFeatures.get(featId);
-                 if(curVal == null) {
-                   docFeatures.put(featId, score);
-                 }
-                 else {
-                   docFeatures.put(featId, curVal + score);
+                 if(!docFeatures.containsKey(featId)) {
+                   docFeatures.put(featId, operators.get(modelName + "-" + paramId).newInstance());
                  }
+                 docFeatures.get(featId).addScore(score);
                }
              }
            }
@@ -284,27 +301,24 @@ private void extract() throws Exception {
            double score = c.getPotential();
 
            // update feature values
-           Double curVal = docFeatures.get(featId);
-           if(curVal == null) {
-             docFeatures.put(featId, score);
+           if(!docFeatures.containsKey(featId)) {
+             docFeatures.put(featId, operators.get(featId).newInstance());
            }
-           else {
-             docFeatures.put(featId, curVal + score);
-           }
-
+           docFeatures.get(featId).addScore(score);
          }
        }
      }
 
      // print feature values for current query
-     for(Entry<String, SortedMap<String, Double>> featureEntry : featureValues.entrySet()) {
+     for(Entry<String, SortedMap<String, Operator>> featureEntry : featureValues.entrySet()) {
        String docName = featureEntry.getKey();
        System.out.print(qid + "\t" + docName);
-       Map<String,Double> docFeatures = featureEntry.getValue();
+       Map<String,Operator> docFeatures = featureEntry.getValue();
        for(String featureName : featureNames) {
-         Double featVal = docFeatures.get(featureName);
-         if(featVal == null) {
-           featVal = DEFAULT_FEATURE_VALUE;
+         Operator op = docFeatures.get(featureName);
+         double featVal = DEFAULT_FEATURE_VALUE;
+         if(op != null) {
+           featVal = op.getFinalScore();
          }
          System.out.print("\t" + featVal);
        }

diff --git a/src/java/main/ivory/ltr/operator/BooleanCount.java b/src/java/main/ivory/ltr/operator/BooleanCount.java
@@ -0,0 +1,17 @@
+package ivory.ltr.operator;
+
+public class BooleanCount extends Operator {
+  @Override public double getFinalScore() {
+    int s = 0;
+    for(double f: scores) {
+      if(f > 0) {
+        s++;
+      }
+    }
+    return ((double) s);
+  }
+
+  @Override public Operator newInstance() {
+    return new BooleanCount();
+  }
+}
diff --git a/src/java/main/ivory/ltr/operator/BooleanRatio.java b/src/java/main/ivory/ltr/operator/BooleanRatio.java
@@ -0,0 +1,21 @@
+package ivory.ltr.operator;
+
+public class BooleanRatio extends Operator {
+  @Override public double getFinalScore() {
+    if(scores.size() == 0) {
+      return 0;
+    }
+
+    int s = 0;
+    for(double f: scores) {
+      if(f > 0) {
+        s++;
+      }
+    }
+    return ((double) s) / scores.size();
+  }
+
+  @Override public Operator newInstance() {
+    return new BooleanRatio();
+  }
+}
diff --git a/src/java/main/ivory/ltr/operator/Max.java b/src/java/main/ivory/ltr/operator/Max.java
@@ -0,0 +1,17 @@
+package ivory.ltr.operator;
+
+public class Max extends Operator {
+  @Override public double getFinalScore() {
+    double s = Double.NEGATIVE_INFINITY;
+    for(double f: scores) {
+      if(f > s) {
+        s = f;
+      }
+    }
+    return s;
+  }
+
+  @Override public Operator newInstance() {
+    return new Max();
+  }
+}
diff --git a/src/java/main/ivory/ltr/operator/Mean.java b/src/java/main/ivory/ltr/operator/Mean.java
@@ -0,0 +1,20 @@
+package ivory.ltr.operator;
+
+public class Mean extends Operator {
+  @Override public double getFinalScore() {
+    if(scores.size() == 0) {
+      return 0;
+    }
+
+    double s = 0;
+    for(double f: scores) {
+      s += f;
+    }
+
+    return (s / scores.size());
+  }
+
+  @Override public Operator newInstance() {
+    return new Mean();
+  }
+}
diff --git a/src/java/main/ivory/ltr/operator/Min.java b/src/java/main/ivory/ltr/operator/Min.java
@@ -0,0 +1,17 @@
+package ivory.ltr.operator;
+
+public class Min extends Operator {
+  @Override public double getFinalScore() {
+    double s = Double.POSITIVE_INFINITY;
+    for(double f: scores) {
+      if(f < s) {
+        s = f;
+      }
+    }
+    return s;
+  }
+
+  @Override public Operator newInstance() {
+    return new Min();
+  }
+}
diff --git a/src/java/main/ivory/ltr/operator/Operator.java b/src/java/main/ivory/ltr/operator/Operator.java
@@ -0,0 +1,46 @@
+package ivory.ltr.operator;
+
+import java.util.List;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Holds and computes the final value of a feature
+ *
+ * @author Nima Asadi
+ */
+public abstract class Operator {
+  protected List<Double> scores;
+
+  protected Operator() {
+    scores = Lists.newArrayList();
+  }
+
+  /**
+   * Adds a new score
+   *
+   * @param score Score
+   */
+  public void addScore(double score) {
+    scores.add(score);
+  }
+
+  /**
+   * Clears the scores
+   */
+  public void clear() {
+    scores.clear();
+  }
+
+  /**
+   * Computes the final feature value
+   *
+   * @return Feature value
+   */
+  public abstract double getFinalScore();
+
+  /**
+   * @return New instance
+   */
+  public abstract Operator newInstance();
+}