Permalink
Browse files

Merge branch 'master' of https://github.com/linkedin/sensei into inde…

…x_pruner
  • Loading branch information...
2 parents 4cd49fd + f256da3 commit 1da9e4267e83a51a798cf36aefa26ad635198778 Rahul Aggarwal committed Feb 29, 2012
@@ -8,8 +8,15 @@
public class RoutingInfo
{
+ // Three parallel arrays
+
+ /** Index to partition Id */
public final int[] partitions;
+
+ /** Which node to use in each partition */
public final int[] nodegroup;
+
+ /** List of nodes for each partition */
public final List<Node>[] nodelist;
public RoutingInfo(final List<Node>[] nodelist, int[] partitions, int[] nodegroup)
@@ -24,5 +31,12 @@ public String toString()
StringBuffer sb = new StringBuffer();
sb.append("Nodes: ").append(Arrays.toString(nodegroup)).append(" each for partitions: ").append(Arrays.toString(partitions));
return sb.toString();
+//
+// StringBuilder builder = new StringBuilder("Nodes: ");
+// for (int i = 0; i < partitions.length; i++) {
+// builder.append(String.format("p%d:%d", partitions[i], nodelist[i].get(nodegroup[i]).getId())).append(" ");
+// }
+//
+// return builder.toString();
}
}
@@ -16,16 +16,12 @@ public MFacetDouble(MultiValueFacetDataCache mDataCaches)
@Override
public boolean containsAll(Set set)
{
- for(int i=0; i< this.length; i++)
- if(set.contains(((TermDoubleList) _mTermList).getPrimitiveValue(buf[i])))
- return true;
-
- return false;
+ throw new UnsupportedOperationException("not implemented yet");
}
public boolean containsAll(double[] target)
{
- return false;
+ throw new UnsupportedOperationException("not implemented yet");
}
@@ -16,16 +16,12 @@ public MFacetFloat(MultiValueFacetDataCache mDataCaches)
@Override
public boolean containsAll(Set set)
{
- for(int i=0; i< this.length; i++)
- if(set.contains(((TermFloatList) _mTermList).getPrimitiveValue(buf[i])))
- return true;
-
- return false;
+ throw new UnsupportedOperationException("not implemented yet");
}
public boolean containsAll(float[] target)
{
- return false;
+ throw new UnsupportedOperationException("not implemented yet");
}
@@ -16,16 +16,12 @@ public MFacetInt(MultiValueFacetDataCache mDataCaches)
@Override
public boolean containsAll(Set set)
{
- for(int i=0; i< this.length; i++)
- if(set.contains(((TermIntList) _mTermList).getPrimitiveValue(buf[i])))
- return true;
-
- return false;
+ throw new UnsupportedOperationException("not implemented yet");
}
public boolean containsAll(int[] target)
{
- return false;
+ throw new UnsupportedOperationException("not implemented yet");
}
@@ -16,16 +16,12 @@ public MFacetLong(MultiValueFacetDataCache mDataCaches)
@Override
public boolean containsAll(Set set)
{
- for(int i=0; i< this.length; i++)
- if(set.contains(((TermLongList) _mTermList).getPrimitiveValue(buf[i])))
- return true;
-
- return false;
+ throw new UnsupportedOperationException("not implemented yet");
}
public boolean containsAll(long[] target)
{
- return false;
+ throw new UnsupportedOperationException("not implemented yet");
}
@@ -16,17 +16,12 @@ public MFacetShort(MultiValueFacetDataCache mDataCaches)
@Override
public boolean containsAll(Set set)
{
- for(int i=0; i< this.length; i++)
- if(set.contains(((TermShortList) _mTermList).getPrimitiveValue(buf[i])))
- return true;
-
- return false;
+ throw new UnsupportedOperationException("not implemented yet");
}
public boolean containsAll(short[] target)
{
-
- return false;
+ throw new UnsupportedOperationException("not implemented yet");
}
@@ -16,16 +16,12 @@ public MFacetString(MultiValueFacetDataCache mDataCaches)
@Override
public boolean containsAll(Set set)
{
- for(int i=0; i< this.length; i++)
- if(set.contains(((TermStringList) _mTermList).get(buf[i])))
- return true;
-
- return false;
+ throw new UnsupportedOperationException("not implemented yet");
}
public boolean containsAll(String[] target)
{
- return false;
+ throw new UnsupportedOperationException("not implemented yet");
}
@@ -12,6 +12,12 @@
import it.unimi.dsi.fastutil.ints.Int2LongOpenHashMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
+import it.unimi.dsi.fastutil.objects.Object2FloatOpenHashMap;
+import it.unimi.dsi.fastutil.objects.Object2DoubleOpenHashMap;
+import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap;
+import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -82,6 +88,12 @@
public static final String KW_TYPE_MAP_INT_DOUBLE = "map_int_double";
public static final String KW_TYPE_MAP_INT_LONG = "map_int_long";
+ public static final String KW_TYPE_MAP_STRING_INT = "map_string_int";
+ public static final String KW_TYPE_MAP_STRING_FLOAT = "map_string_float";
+ public static final String KW_TYPE_MAP_STRING_STRING = "map_string_string";
+ public static final String KW_TYPE_MAP_STRING_DOUBLE = "map_string_double";
+ public static final String KW_TYPE_MAP_STRING_LONG = "map_string_long";
+
// normal type: [int, double, float, long, bool, string]
public static final String KW_TYPE_INT = "int";
public static final String KW_TYPE_FLOAT = "float";
@@ -139,6 +151,11 @@
private final String TYPE_MAP_INT_DOUBLE = "MAP_INT_DOUBLE";
private final String TYPE_MAP_INT_FLOAT = "MAP_INT_FLOAT";
private final String TYPE_MAP_INT_STRING = "MAP_INT_STRING";
+ private final String TYPE_MAP_STRING_INT = "MAP_STRING_INT";
+ private final String TYPE_MAP_STRING_LONG = "MAP_STRING_LONG";
+ private final String TYPE_MAP_STRING_DOUBLE = "MAP_STRING_DOUBLE";
+ private final String TYPE_MAP_STRING_FLOAT = "MAP_STRING_FLOAT";
+ private final String TYPE_MAP_STRING_STRING = "MAP_STRING_STRING";
private final String TYPE_MAP_HEAD = "MAP";
@@ -197,6 +214,8 @@
private static final long serialVersionUID = 1L;
+ public static int MAX_NUM_MODELS = 10000;
+
private static Logger logger = Logger.getLogger(RelevanceQuery.class);
protected final Query _query;
@@ -235,6 +254,14 @@
pool.importPackage("it.unimi.dsi.fastutil.ints.Int2LongOpenHashMap");
pool.importPackage("it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap");
+ pool.importPackage("it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap");
+ pool.importPackage("it.unimi.dsi.fastutil.objects.Object2FloatOpenHashMap");
+ pool.importPackage("it.unimi.dsi.fastutil.objects.Object2DoubleOpenHashMap");
+ pool.importPackage("it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap");
+ pool.importPackage("it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap");
+
+ pool.importPackage("it.unimi.dsi.fastutil.objects.AbstractObject2FloatMap");
+
pool.importPackage("com.senseidb.search.relevance.MFacet");
pool.importPackage("com.senseidb.search.relevance.MFacetDouble");
@@ -268,6 +295,14 @@
hs_safe.add("it.unimi.dsi.fastutil.ints.Int2LongOpenHashMap");
hs_safe.add("it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap");
+ hs_safe.add("it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap");
+ hs_safe.add("it.unimi.dsi.fastutil.objects.Object2FloatOpenHashMap");
+ hs_safe.add("it.unimi.dsi.fastutil.objects.Object2DoubleOpenHashMap");
+ hs_safe.add("it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap");
+ hs_safe.add("it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap");
+
+ hs_safe.add("it.unimi.dsi.fastutil.objects.AbstractObject2FloatMap");
+
hs_safe.add("com.senseidb.search.relevance.RelevanceQuery");
hs_safe.add("com.senseidb.search.relevance.CustomScorer");
@@ -312,7 +347,7 @@
//
// "variables": {
// "set_int":["c","d"], // supported hashset types: [set_int, set_float, set_string, set_double, set_long]
-// "map_int_float":["j"], // currently supported hashmap: [map_int_float, map_int_double, map_int_*...]
+// "map_int_float":["j"], // currently supported hashmap: [map_int_float, map_int_double, map_int_*...] [map_string_int, map_string_float, map_string_*]
// "int":["e","f"], // supported normal variables: [int, double, float, long, bool, string]
// "long":["g","h"]
// },
@@ -367,20 +402,27 @@
"variables":{
"set_int":["goodYear"],
"int":["thisYear"],
- "map_int_float":["mileageWeight"]
+ "map_int_float":["mileageWeight"],
+ "map_int_string":["yearcolor"],
+ "map_string_float":["colorweight"],
+ "map_string_string":["categorycolor"]
},
"facets":{
"int":["year","mileage"],
- "long":["groupid"]
+ "long":["groupid"],
+ "string":["color","category"]
},
- "function_params":["_INNER_SCORE", "thisYear", "year","goodYear","mileageWeight","mileage"],
- "function":" if(mileageWeight.containsKey(mileage)) return 10000+mileageWeight.get(mileage); if(goodYear.contains(year)) return (float)Math.exp(2d); if(year==thisYear) return 87f ; return _INNER_SCORE;"
+ "function_params":["_INNER_SCORE", "thisYear", "year","goodYear","mileageWeight","mileage","color", "yearcolor", "colorweight", "category", "categorycolor"],
+ "function":" if(categorycolor.containsKey(category) && categorycolor.get(category).equals(color)) return 10000f; if(colorweight.containsKey(color) ) return 200f + colorweight.getFloat(color); if(yearcolor.containsKey(year) && yearcolor.get(year).equals(color)) return 200f; if(mileageWeight.containsKey(mileage)) return 10000+mileageWeight.get(mileage); if(goodYear.contains(year)) return (float)Math.exp(2d); if(year==thisYear) return 87f ; return _INNER_SCORE;"
},
"values":{
"goodYear":[1996,1997],
"thisYear":2001,
- "mileageWeight":{"key":[11400,11000],"value":[777.9, 10.2]}
+ "mileageWeight":{"key":[11400,11000],"value":[777.9, 10.2]},
+ "yearcolor":{"key":[1998],"value":["red"]},
+ "colorweight":{"key":["red"],"value":[335.5]},
+ "categorycolor":{"key":["compact"],"value":["red"]}
}
}
}
@@ -596,6 +638,47 @@ else if (KW_TYPE_MAP_INT_STRING.equals(type))
((Int2ObjectOpenHashMap)hm).put(keysList.getInt(j), valuesList.getString(j));
hm_type.put(symbol, TYPE_MAP_INT_STRING);
}
+
+ else if(KW_TYPE_MAP_STRING_INT.equals(type))
+ {
+ if(hm == null)
+ hm = new Object2IntOpenHashMap();
+ for(int j=0; j<keySize; j++)
+ ((Object2IntOpenHashMap)hm).put(keysList.getString(j), valuesList.getInt(j));
+ hm_type.put(symbol, TYPE_MAP_STRING_INT);
+ }
+ else if (KW_TYPE_MAP_STRING_DOUBLE.equals(type))
+ {
+ if(hm == null)
+ hm = new Object2DoubleOpenHashMap();
+ for(int j=0; j<keySize; j++)
+ ((Object2DoubleOpenHashMap)hm).put(keysList.getString(j), valuesList.getDouble(j));
+ hm_type.put(symbol, TYPE_MAP_STRING_DOUBLE);
+ }
+ else if (KW_TYPE_MAP_STRING_FLOAT.equals(type))
+ {
+ if(hm == null)
+ hm = new Object2FloatOpenHashMap();
+ for(int j=0; j<keySize; j++)
+ ((Object2FloatOpenHashMap)hm).put(keysList.getString(j), (float)(valuesList.getDouble(j)));
+ hm_type.put(symbol, TYPE_MAP_STRING_FLOAT);
+ }
+ else if (KW_TYPE_MAP_STRING_LONG.equals(type))
+ {
+ if(hm == null)
+ hm = new Object2LongOpenHashMap();
+ for(int j=0; j<keySize; j++)
+ ((Object2LongOpenHashMap)hm).put(keysList.getString(j), valuesList.getLong(j));
+ hm_type.put(symbol, TYPE_MAP_STRING_LONG);
+ }
+ else if (KW_TYPE_MAP_STRING_STRING.equals(type))
+ {
+ if(hm == null)
+ hm = new Object2ObjectOpenHashMap();
+ for(int j=0; j<keySize; j++)
+ ((Object2ObjectOpenHashMap)hm).put(keysList.getString(j), valuesList.getString(j));
+ hm_type.put(symbol, TYPE_MAP_STRING_STRING);
+ }
if(hm_var.containsKey(symbol))
@@ -798,6 +881,9 @@ else if(KW_TYPE_BOOL.equals(type))
throw new JSONException(e);
}
+ if(hmModels.size() > MAX_NUM_MODELS)
+ hmModels = new HashMap<String, CustomScorer>();
+
hmModels.put(className, cscorer);
}
}
@@ -961,7 +1047,6 @@ private String makeFuncString(String funcBody,
HashMap<String, String> hm_type,
LinkedList<String> lls_params) throws JSONException
{
-// "public float score(Object[] objs) { Integer inta = (Integer)objs[0]; System.out.println(inta); HashMap hm = (HashMap)objs[2]; System.out.println(hm.get(\"good\")); return b; }"
StringBuffer sb = new StringBuffer();
sb.append("public float score(short[] shorts, int[] ints, long[] longs, float[] floats, double[] doubles, boolean[] booleans, String[] strings, Set[] sets, Map[] maps, com.senseidb.search.relevance.MFacetInt[] mFacetInts, com.senseidb.search.relevance.MFacetLong[] mFacetLongs, com.senseidb.search.relevance.MFacetFloat[] mFacetFloats, com.senseidb.search.relevance.MFacetDouble[] mFacetDoubles, com.senseidb.search.relevance.MFacetShort[] mFacetShorts, com.senseidb.search.relevance.MFacetString[] mFacetStrings) {");
@@ -1076,6 +1161,32 @@ else if(paramType.equals(TYPE_MAP_INT_STRING))
map_index++;
}
+ else if(paramType.equals(TYPE_MAP_STRING_INT))
+ {
+ sb.append(" it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap " + paramName + " = (it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap) maps["+ map_index +"]; ");
+ map_index++;
+ }
+ else if(paramType.equals(TYPE_MAP_STRING_LONG))
+ {
+ sb.append(" it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap " + paramName + " = (it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap) maps["+ map_index +"]; ");
+ map_index++;
+ }
+ else if(paramType.equals(TYPE_MAP_STRING_DOUBLE))
+ {
+ sb.append(" it.unimi.dsi.fastutil.objects.Object2DoubleOpenHashMap " + paramName + " = (it.unimi.dsi.fastutil.objects.Object2DoubleOpenHashMap) maps["+ map_index +"]; ");
+ map_index++;
+ }
+ else if(paramType.equals(TYPE_MAP_STRING_FLOAT))
+ {
+ sb.append(" it.unimi.dsi.fastutil.objects.Object2FloatOpenHashMap " + paramName + " = (it.unimi.dsi.fastutil.objects.Object2FloatOpenHashMap) maps["+ map_index +"]; ");
+ map_index++;
+ }
+ else if(paramType.equals(TYPE_MAP_STRING_STRING))
+ {
+ sb.append(" it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap " + paramName + " = (it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap) maps["+ map_index +"]; ");
+ map_index++;
+ }
+
// innerscore
else if(paramType.equals(TYPE_INNER_SCORE))
{
@@ -1613,7 +1724,6 @@ public float score() throws IOException {
}
}
-// float score(short[] shorts, int[] ints, long[] longs, float[] floats, double[] doubles, boolean[] booleans, String[] strings, Set[] sets);
return _cscorer.score(shorts, ints, longs, floats, doubles, booleans, strings, sets, maps, mFacetInts, mFacetLongs, mFacetFloats, mFacetDoubles, mFacetShorts, mFacetStrings);
}
@@ -473,6 +473,8 @@ private static String readContent(BufferedReader reader) throws IOException{
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
+ if (req.getCharacterEncoding() == null)
+ req.setCharacterEncoding("UTF-8");
resp.setContentType("application/json; charset=utf-8");
resp.setCharacterEncoding("UTF-8");

0 comments on commit 1da9e42

Please sign in to comment.