Skip to content
Browse files

Enhanced BQL relevance model support:

1. Automatically detect facets and internal variables used in relevance
   models, so users do not need to specify them in the model parameter
   list.

2. Added code to detect and handle different variable declaration
   errors.
  • Loading branch information...
1 parent c1e5b91 commit 2a6e942604f902de60a605c121e7e35bc97a719e Baoqiu Cui committed
View
1 sensei-core/.project
@@ -1,3 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>sensei-core</name>
<comment>sensei core search. NO_M2ECLIPSE_SUPPORT: Project files created with the maven-eclipse-plugin are not supported in M2Eclipse.</comment>
View
266 sensei-core/src/main/antlr3/com/senseidb/bql/parsers/BQL.g
@@ -264,8 +264,10 @@ package com.senseidb.bql.parsers;
@parser::header {
package com.senseidb.bql.parsers;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.Map;
import java.util.Set;
import org.json.JSONObject;
@@ -283,7 +285,8 @@ import java.text.SimpleDateFormat;
private static final int DEFAULT_FACET_MINHIT = 1;
private static final int DEFAULT_FACET_MAXHIT = 10;
private static final Map<String, String> _fastutilTypeMap;
- private static final Set<String> _builtinRelevanceVars;
+ private static final Map<String, String> _internalVarMap;
+ private static final Set<String> _supportedClasses;
private Map<String, String[]> _facetInfoMap;
private long _now;
@@ -291,6 +294,11 @@ import java.text.SimpleDateFormat;
private SimpleDateFormat[] _format1 = new SimpleDateFormat[2];
private SimpleDateFormat[] _format2 = new SimpleDateFormat[2];
+ private LinkedList<Map<String, String>> _symbolTable;
+ private Map<String, String> _currentScope;
+ private Set<String> _usedFacets; // Facets used by relevance model
+ private Set<String> _usedInternalVars; // Internal variables used by relevance model
+
static {
_fastutilTypeMap = new HashMap<String, String>();
_fastutilTypeMap.put("IntOpenHashSet", "set_int");
@@ -311,9 +319,23 @@ import java.text.SimpleDateFormat;
_fastutilTypeMap.put("Object2LongOpenHashMap", "map_string_long");
_fastutilTypeMap.put("Object2ObjectOpenHashMap", "map_string_string");
- _builtinRelevanceVars = new HashSet<String>();
- _builtinRelevanceVars.add("_NOW");
- _builtinRelevanceVars.add("_INNER_SCORE");
+ _internalVarMap = new HashMap<String, String>();
+ _internalVarMap.put("_NOW", "long");
+ _internalVarMap.put("_INNER_SCORE", "float");
+
+ _supportedClasses = new HashSet<String>();
+ _supportedClasses.add("Boolean");
+ _supportedClasses.add("Byte");
+ _supportedClasses.add("Character");
+ _supportedClasses.add("Double");
+ _supportedClasses.add("Integer");
+ _supportedClasses.add("Long");
+ _supportedClasses.add("Short");
+
+ _supportedClasses.add("Math");
+ _supportedClasses.add("String");
+ _supportedClasses.add("System");
+
}
public BQLParser(TokenStream input, Map<String, String[]> facetInfoMap)
@@ -684,57 +706,62 @@ import java.text.SimpleDateFormat;
new JSONObject().put(field, newSpec)));
}
- private void processRelevanceModelParam(JSONObject json,
+ private void processRelevanceModelParam(TokenStream input,
+ JSONObject json,
+ Set<String> params,
String typeName,
final String varName)
- throws JSONException
+ throws JSONException, RecognitionException
{
+ if (_facetInfoMap.containsKey(varName)) {
+ throw new FailedPredicateException(input, "", "Facet name \"" + varName + "\" cannot be used as a relevance model parameter.");
+ }
+
+ if (_internalVarMap.containsKey(varName)) {
+ throw new FailedPredicateException(input, "", "Internal variable \"" + varName + "\" cannot be used as a relevance model parameter.");
+ }
+
+ if (params.contains(varName)) {
+ throw new FailedPredicateException(input, "", "Parameter name \"" + varName + "\" has already been used.");
+ }
+
+ if ("String".equals(typeName)) {
+ typeName = "string";
+ }
+
JSONArray funcParams = json.optJSONArray("function_params");
if (funcParams == null) {
funcParams = new JSONArray();
json.put("function_params", funcParams);
}
- // XXX Need to detect duplicates
funcParams.put(varName);
+ params.add(varName);
- if ("String".equals(typeName)) {
- typeName = "string";
+ JSONObject variables = json.optJSONObject("variables");
+ if (variables == null) {
+ variables = new JSONObject();
+ json.put("variables", variables);
}
- if (_builtinRelevanceVars.contains(varName)) {
- // Do nothing
+ JSONArray varsWithSameType = variables.optJSONArray(typeName);
+ if (varsWithSameType == null) {
+ varsWithSameType = new JSONArray();
+ variables.put(typeName, varsWithSameType);
}
- else if (_facetInfoMap.get(varName) == null) {
- // This is NOT a facet, put it in the variable list
- JSONObject variables = json.optJSONObject("variables");
- if (variables == null) {
- variables = new JSONObject();
- json.put("variables", variables);
- }
-
- JSONArray varsWithSameType = variables.optJSONArray(typeName);
- if (varsWithSameType == null) {
- varsWithSameType = new JSONArray();
- variables.put(typeName, varsWithSameType);
- }
- varsWithSameType.put(varName);
- }
- else {
- JSONObject facets = json.optJSONObject("facets");
- if (facets == null) {
- facets = new JSONObject();
- json.put("facets", facets);
- }
+ varsWithSameType.put(varName);
+ }
- // XXX Need to double check the type
- JSONArray facetsWithSameType = facets.optJSONArray(typeName);
- if (facetsWithSameType == null) {
- facetsWithSameType = new JSONArray();
- facets.put(typeName, facetsWithSameType);
+ // Check whether a variable is defined.
+ private boolean verifyVariable(final String variable) {
+ Iterator<Map<String, String>> itr = _symbolTable.descendingIterator();
+ while (itr.hasNext()) {
+ Map<String, String> scope = itr.next();
+ if (scope.containsKey(variable)) {
+ return true;
}
- facetsWithSameType.put(varName);
}
+ return false;
}
}
@@ -2157,12 +2184,26 @@ given_clause returns [JSONObject json]
// Relevance model related
// =====================================================================
-variable_declarators
- : variable_declarator (',' variable_declarator)*
+variable_declarators returns [JSONArray json]
+@init {
+ $json = new JSONArray();
+}
+ : var1=variable_declarator
+ {
+ $json.put($var1.varName);
+ }
+ (COMMA var2=variable_declarator
+ {
+ $json.put($var2.varName);
+ }
+ )*
;
-variable_declarator
+variable_declarator returns [String varName]
: variable_declarator_id ('=' variable_initializer)?
+ {
+ $varName = $variable_declarator_id.varName;
+ }
;
variable_declarator_id returns [String varName]
@@ -2227,11 +2268,12 @@ formal_parameters returns [JSONObject json]
formal_parameter_decls returns [JSONObject json]
@init {
$json = new JSONObject();
+ Set<String> params = new HashSet<String>();
}
: decl=formal_parameter_decl
{
try {
- processRelevanceModelParam($json, $decl.typeName, $decl.varName);
+ processRelevanceModelParam(input, $json, params, $decl.typeName, $decl.varName);
}
catch (JSONException err) {
throw new FailedPredicateException(input,
@@ -2242,7 +2284,7 @@ formal_parameter_decls returns [JSONObject json]
(COMMA decl=formal_parameter_decl
{
try {
- processRelevanceModelParam($json, $decl.typeName, $decl.varName);
+ processRelevanceModelParam(input, $json, params, $decl.typeName, $decl.varName);
}
catch (JSONException err) {
throw new FailedPredicateException(input,
@@ -2284,7 +2326,9 @@ boxed_type
;
limited_type
- : { "String".equals(input.LT(1).getText()) }? STRING
+ : 'String'
+ | 'System'
+ | 'Math'
;
variable_modifier
@@ -2292,10 +2336,66 @@ variable_modifier
;
relevance_model returns [String functionBody, JSONObject json]
- : DEFINED AS params=formal_parameters BEGIN model_block END
+@init {
+ _usedFacets = new HashSet<String>();
+ _usedInternalVars = new HashSet<String>();
+ _symbolTable = new LinkedList<Map<String, String>>();
+ _currentScope = new HashMap<String, String>();
+ _symbolTable.offerLast(_currentScope);
+}
+ : DEFINED AS params=formal_parameters
+ {
+ try {
+ JSONObject varParams = $params.json.optJSONObject("variables");
+ if (varParams != null) {
+ Iterator<String> itr = varParams.keys();
+ while (itr.hasNext()) {
+ String key = (String) itr.next();
+ JSONArray vars = varParams.getJSONArray(key);
+ for (int i = 0; i < vars.length(); ++i) {
+ _currentScope.put(vars.getString(i), key);
+ }
+ }
+ }
+ }
+ catch (JSONException err) {
+ throw new FailedPredicateException(input, "relevance_model", "JSONException: " + err.getMessage());
+ }
+ }
+ BEGIN model_block END
{
$functionBody = $model_block.text;
$json = $params.json;
+
+ // Append facets and internal variable to "function_params".
+ try {
+ JSONArray funcParams = $json.getJSONArray("function_params");
+
+ JSONObject facets = new JSONObject();
+ $json.put("facets", facets);
+
+ for (String facet: _usedFacets) {
+ funcParams.put(facet);
+ String typeName = _facetInfoMap.get(facet)[1];
+ JSONArray facetsWithSameType = facets.optJSONArray(typeName);
+ if (facetsWithSameType == null) {
+ facetsWithSameType = new JSONArray();
+ facets.put(typeName, facetsWithSameType);
+ }
+ facetsWithSameType.put(facet);
+ }
+
+ // Internal variables, like _NOW, do not need to be
+ // included in "variables".
+ for (String varName: _usedInternalVars) {
+ funcParams.put(varName);
+ }
+ }
+ catch (JSONException err) {
+ throw new FailedPredicateException(input,
+ "formal_parameter_decl",
+ "JSONException: " + err.getMessage());
+ }
}
;
@@ -2304,7 +2404,17 @@ model_block
;
block
- : '{' block_statement* '}'
+ : '{'
+ {
+ _currentScope = new HashMap<String, String>();
+ _symbolTable.offerLast(_currentScope);
+ }
+ block_statement*
+ {
+ _symbolTable.pollLast();
+ _currentScope = _symbolTable.peekLast();
+ }
+ '}'
;
block_statement
@@ -2318,6 +2428,35 @@ local_variable_declaration_stmt
local_variable_declaration
: variable_modifiers type variable_declarators
+ {
+ try {
+ JSONArray vars = $variable_declarators.json;
+ for (int i = 0; i < vars.length(); ++i) {
+ String var = vars.getString(i);
+ if (_facetInfoMap.containsKey(var)) {
+ throw new FailedPredicateException(input,
+ "local_variable_declaration",
+ "Facet name \"" + var + "\" cannot be used to declare a variable.");
+ }
+ else if (_internalVarMap.containsKey(var)) {
+ throw new FailedPredicateException(input,
+ "local_variable_declaration",
+ "Internal variable \"" + var + "\" cannot be re-used to declare another variable.");
+ }
+ else if (verifyVariable(var)) {
+ throw new FailedPredicateException(input,
+ "local_variable_declaration",
+ "Variable \"" + var + "\" is already defined.");
+ }
+ else {
+ _currentScope.put(var, $type.typeName);
+ }
+ }
+ }
+ catch (JSONException err) {
+ throw new FailedPredicateException(input, "local_variable_declaration", "JSONException: " + err.getMessage());
+ }
+ }
;
variable_modifiers
@@ -2327,7 +2466,16 @@ variable_modifiers
java_statement
: block
| 'if' par_expression java_statement (else_statement)?
- | 'for' LPAR for_control RPAR java_statement
+ | 'for' LPAR
+ {
+ _currentScope = new HashMap<String, String>();
+ _symbolTable.offerLast(_currentScope);
+ }
+ for_control RPAR java_statement
+ {
+ _symbolTable.pollLast();
+ _currentScope = _symbolTable.peekLast();
+ }
| 'while' par_expression java_statement
| 'do' java_statement 'while' par_expression SEMI
| 'switch' par_expression '{' switch_block_statement_groups '}'
@@ -2335,7 +2483,7 @@ java_statement
| 'break' IDENT? SEMI
| 'continue' IDENT? SEMI
| SEMI
- | statement_expression SEMI
+ | statement_expression SEMI
;
else_statement
@@ -2524,7 +2672,27 @@ cast_expression
primary
: par_expression
| literal
- | IDENT ('.' java_method)* identifier_suffix?
+ | java_ident ('.' java_method)* identifier_suffix?
+ {
+ String var = $java_ident.text;
+ if (_facetInfoMap.containsKey(var)) {
+ _usedFacets.add(var);
+ }
+ else if (_internalVarMap.containsKey(var)) {
+ _usedInternalVars.add(var);
+ }
+ else if (!_supportedClasses.contains(var) && !verifyVariable(var)) {
+ throw new FailedPredicateException(input,
+ "primary",
+ "Variable or class \"" + var + "\" is not defined.");
+ }
+ }
+ ;
+
+java_ident
+ : boxed_type
+ | limited_type
+ | IDENT
;
// Need to handle the conflicts of BQL keywords and common Java method
View
2 sensei-core/src/test/java/com/senseidb/test/TestSensei.java
@@ -252,7 +252,7 @@ public void testBqlExtraFilter() throws Exception
public void testBqlRelevance1() throws Exception
{
logger.info("Executing test case testBqlRelevance1");
- String req = "{\"bql\":\"SELECT * FROM cars USING RELEVANCE MODEL my_model ('thisYear':2001, 'goodYear':(1996)) DEFINED AS (float _INNER_SCORE, int thisYear, int year, IntOpenHashSet goodYear) BEGIN if (goodYear.contains(year)) return (float)Math.exp(10d); if (year==thisYear) return 87f; return _INNER_SCORE; END\"}";
+ String req = "{\"bql\":\"SELECT * FROM cars USING RELEVANCE MODEL my_model ('thisYear':2001, 'goodYear':(1996)) DEFINED AS (int thisYear, IntOpenHashSet goodYear) BEGIN if (goodYear.contains(year)) return (float)Math.exp(10d); if (year==thisYear) return 87f; return _INNER_SCORE; END\"}";
JSONObject res = search(new JSONObject(req));
View
81 sensei-core/src/test/java/com/senseidb/test/bql/parsers/TestBQL.java
@@ -1299,9 +1299,10 @@ public void testRelevanceModelIfStmt() throws Exception
"FROM cars " +
"WHERE color = 'red' " +
"USING RELEVANCE MODEL my_model ('srcid':1234) " +
- " DEFINED AS (float _INNER_SCORE, int intParam1, int intParam2, String strParam) " +
+ " DEFINED AS (int intParam1, int intParam2, String strParam, int srcid) " +
" BEGIN " +
- " int myInt = 100; " +
+ " int myInt = 100 + intParam1 + intParam2; " +
+ " String newStr = strParam; " +
" if (srcid == myInt + 2) " +
" return 123; " +
" else if (srcid > 200) " +
@@ -1311,7 +1312,7 @@ public void testRelevanceModelIfStmt() throws Exception
" END "
);
- JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"_INNER_SCORE\",\"intParam1\",\"intParam2\",\"strParam\"],\"variables\":{\"int\":[\"intParam1\",\"intParam2\"],\"string\":[\"strParam\"]},\"function\":\"int myInt = 100; if (srcid == myInt + 2) return 123; else if (srcid > 200) return 345; else return _INNER_SCORE;\"},\"values\":{\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
+ JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"intParam1\",\"intParam2\",\"strParam\",\"srcid\",\"_INNER_SCORE\"],\"facets\":{},\"variables\":{\"int\":[\"intParam1\",\"intParam2\",\"srcid\"],\"string\":[\"strParam\"]},\"function\":\"int myInt = 100 + intParam1 + intParam2; String newStr = strParam; if (srcid == myInt + 2) return 123; else if (srcid > 200) return 345; else return _INNER_SCORE;\"},\"values\":{\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
assertTrue(_comp.isEquals(json, expected));
}
@@ -1329,7 +1330,7 @@ public void testRelevanceModelFloatLiteral() throws Exception
" DEFINED AS (int srcid) " +
" BEGIN " +
" float x1 = 1.2; " +
- " x = 7.; " +
+ " int x = 7.; " +
" x = 7.5e12; " +
" x = 7.5e-12; " +
" x = 7.5e-12f; " +
@@ -1340,7 +1341,7 @@ public void testRelevanceModelFloatLiteral() throws Exception
" END "
);
- JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"srcid\"],\"variables\":{\"int\":[\"srcid\"]},\"function\":\"float x1 = 1.2; x = 7.; x = 7.5e12; x = 7.5e-12; x = 7.5e-12f; x = .34; x = .34e+12; x = 123f; return 0.25;\"},\"values\":{\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
+ JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"srcid\"],\"facets\":{},\"variables\":{\"int\":[\"srcid\"]},\"function\":\"float x1 = 1.2; int x = 7.; x = 7.5e12; x = 7.5e-12; x = 7.5e-12f; x = .34; x = .34e+12; x = 123f; return 0.25;\"},\"values\":{\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
assertTrue(_comp.isEquals(json, expected));
}
@@ -1372,7 +1373,7 @@ public void testRelevanceModelDataTypes() throws Exception
" END "
);
- JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"intParam1\",\"intParam2\",\"strParam\"],\"variables\":{\"string\":[\"strParam\"],\"int\":[\"intParam1\",\"intParam2\"]},\"function\":\"int myInt = 100; String str1; String str2 = \\\"abcd\\\"; char ch = 'c'; Integer int1, int2; int int3 = 0L, int4 = 1234l; float f1 = 1.23f, f2 = 1.23F; float e1 = 2e+1234; Byte byte1; IntOpenHashSet mySet1, mySet2; Object2IntOpenHashMap myMap1; return 0.123f;\"},\"values\":{\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
+ JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"intParam1\",\"intParam2\",\"strParam\"],\"facets\":{},\"variables\":{\"int\":[\"intParam1\",\"intParam2\"],\"string\":[\"strParam\"]},\"function\":\"int myInt = 100; String str1; String str2 = \\\"abcd\\\"; char ch = 'c'; Integer int1, int2; int int3 = 0L, int4 = 1234l; float f1 = 1.23f, f2 = 1.23F; float e1 = 2e+1234; Byte byte1; IntOpenHashSet mySet1, mySet2; Object2IntOpenHashMap myMap1; return 0.123f;\"},\"values\":{\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
assertTrue(_comp.isEquals(json, expected));
}
@@ -1398,7 +1399,7 @@ public void testRelevanceModelWhileStmt() throws Exception
" END "
);
- JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"srcid\"],\"variables\":{\"int\":[\"srcid\"]},\"function\":\"int myInt = 100; while (myInt < 200) { myInt++; myInt = myInt + 10; } return 100;\"},\"values\":{\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
+ JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"srcid\"],\"facets\":{},\"variables\":{\"int\":[\"srcid\"]},\"function\":\"int myInt = 100; while (myInt < 200) { myInt++; myInt = myInt + 10; } return 100;\"},\"values\":{\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
assertTrue(_comp.isEquals(json, expected));
}
@@ -1423,7 +1424,7 @@ public void testRelevanceModelDoWhile() throws Exception
" END "
);
- JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"srcid\"],\"variables\":{\"int\":[\"srcid\"]},\"function\":\"int myInt = 100; do { myInt = myInt + 10; } while (myInt < 100); return 100;\"},\"values\":{\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
+ JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"srcid\"],\"facets\":{},\"variables\":{\"int\":[\"srcid\"]},\"function\":\"int myInt = 100; do { myInt = myInt + 10; } while (myInt < 100); return 100;\"},\"values\":{\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
assertTrue(_comp.isEquals(json, expected));
}
@@ -1442,13 +1443,13 @@ public void testRelevanceModelForLoop() throws Exception
" BEGIN " +
" int myInt = 0; " +
" for (int i = 0; i < 100; i++) { " +
- " myInt = myInt + 10; " +
+ " myInt = myInt + i * 10; " +
" } " +
" return myInt; " +
" END "
);
- JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"srcid\"],\"variables\":{\"int\":[\"srcid\"]},\"function\":\"int myInt = 0; for (int i = 0; i < 100; i++) { myInt = myInt + 10; } return myInt;\"},\"values\":{\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
+ JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"srcid\"],\"facets\":{},\"variables\":{\"int\":[\"srcid\"]},\"function\":\"int myInt = 0; for (int i = 0; i < 100; i++) { myInt = myInt + i * 10; } return myInt;\"},\"values\":{\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
assertTrue(_comp.isEquals(json, expected));
}
@@ -1479,7 +1480,7 @@ public void testRelevanceModelSwitchStmt() throws Exception
" END "
);
- JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"srcid\"],\"variables\":{\"int\":[\"srcid\"]},\"function\":\"int myInt = 0; switch (myInt) { case 1: myInt = 2; break; case 2: case 3: myInt = 4; break; default: myInt = 100; } return 0.5f;\"},\"values\":{\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
+ JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"srcid\"],\"facets\":{},\"variables\":{\"int\":[\"srcid\"]},\"function\":\"int myInt = 0; switch (myInt) { case 1: myInt = 2; break; case 2: case 3: myInt = 4; break; default: myInt = 100; } return 0.5f;\"},\"values\":{\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
assertTrue(_comp.isEquals(json, expected));
}
@@ -1493,8 +1494,8 @@ public void testRelevanceModelExpressions() throws Exception
"SELECT color, year " +
"FROM cars " +
"WHERE color = 'red' " +
- "USING RELEVANCE MODEL my_model ('srcid':1234, 'timeVal':9999, '_half_time':8888) " +
- " DEFINED AS (int srcid, long timeVal, long _half_time) " +
+ "USING RELEVANCE MODEL my_model ('srcid':1234, 'timeVal':9999, '_half_time':8888, 'coolTag':'zzz') " +
+ " DEFINED AS (int srcid, long timeVal, long _half_time, String coolTag) " +
" BEGIN " +
" int myInt = 0; " +
" float delta = System.currentTimeMillis() - timeVal; " +
@@ -1509,8 +1510,8 @@ public void testRelevanceModelExpressions() throws Exception
" return timeScore; " +
" END "
);
-
- JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"srcid\",\"timeVal\",\"_half_time\"],\"variables\":{\"int\":[\"srcid\"],\"long\":[\"timeVal\",\"_half_time\"]},\"function\":\"int myInt = 0; float delta = System.currentTimeMillis() - timeVal; float t = delta > 0 ? delta : 0; float numHours = t / (1000 * 3600); float timeScore = (float) Math.exp(-(numHours/_half_time)); if (tags.contains(coolTag)) return 999999; int x = 0; x += 5; x *= 10; return timeScore;\"},\"values\":{\"_half_time\":8888,\"timeVal\":9999,\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
+
+ JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"srcid\",\"timeVal\",\"_half_time\",\"coolTag\",\"tags\"],\"facets\":{\"string\":[\"tags\"]},\"variables\":{\"int\":[\"srcid\"],\"string\":[\"coolTag\"],\"long\":[\"timeVal\",\"_half_time\"]},\"function\":\"int myInt = 0; float delta = System.currentTimeMillis() - timeVal; float t = delta > 0 ? delta : 0; float numHours = t / (1000 * 3600); float timeScore = (float) Math.exp(-(numHours/_half_time)); if (tags.contains(coolTag)) return 999999; int x = 0; x += 5; x *= 10; return timeScore;\"},\"values\":{\"_half_time\":8888,\"timeVal\":9999,\"coolTag\":\"zzz\",\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
assertTrue(_comp.isEquals(json, expected));
}
@@ -1524,26 +1525,25 @@ public void testRelevanceModelParameters() throws Exception
"SELECT color, year " +
"FROM cars " +
"WHERE color = 'red' " +
- "USING RELEVANCE MODEL my_model ('srcid':1234, 'timeVal':9999, '_half_time':8888) " +
+ "USING RELEVANCE MODEL my_model ('srcid':1234) " +
" DEFINED AS (int intParam1, int intParam2, String strParam, " +
- " DoubleOpenHashSet setParam, Int2IntOpenHashMap mapParam, " +
- " float price, String color) " +
+ " DoubleOpenHashSet setParam, Int2IntOpenHashMap mapParam) " +
" BEGIN " +
" int myInt = 0; " +
- " float delta = System.currentTimeMillis() - timeVal; " +
+ " float delta = System.currentTimeMillis() + intParam1 + intParam2 ; " +
" float t = delta > 0 ? delta : 0; " +
" float numHours = t / (1000 * 3600); " +
- " float timeScore = (float) Math.exp(-(numHours/_half_time)); " +
- " if (tags.contains(coolTag)) " +
+ " float timeScore = (float) Math.exp(numHours); " +
+ " if (tags.contains(\"zzz\")) " +
" return 999999; " +
" int x = 0; " +
" x += 5; " +
" x *= 10; " +
- " return timeScore; " +
+ " return timeScore + _INNER_SCORE + price; " +
" END "
);
- JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"intParam1\",\"intParam2\",\"strParam\",\"setParam\",\"mapParam\",\"price\",\"color\"],\"facets\":{\"string\":[\"color\"],\"float\":[\"price\"]},\"variables\":{\"map_int_int\":[\"mapParam\"],\"string\":[\"strParam\"],\"int\":[\"intParam1\",\"intParam2\"],\"set_double\":[\"setParam\"]},\"function\":\"int myInt = 0; float delta = System.currentTimeMillis() - timeVal; float t = delta > 0 ? delta : 0; float numHours = t / (1000 * 3600); float timeScore = (float) Math.exp(-(numHours/_half_time)); if (tags.contains(coolTag)) return 999999; int x = 0; x += 5; x *= 10; return timeScore;\"},\"values\":{\"_half_time\":8888,\"timeVal\":9999,\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
+ JSONObject expected = new JSONObject("{\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"intParam1\",\"intParam2\",\"strParam\",\"setParam\",\"mapParam\",\"tags\",\"price\",\"_INNER_SCORE\"],\"facets\":{\"string\":[\"tags\"],\"float\":[\"price\"]},\"variables\":{\"map_int_int\":[\"mapParam\"],\"int\":[\"intParam1\",\"intParam2\"],\"string\":[\"strParam\"],\"set_double\":[\"setParam\"]},\"function\":\"int myInt = 0; float delta = System.currentTimeMillis() + intParam1 + intParam2 ; float t = delta > 0 ? delta : 0; float numHours = t / (1000 * 3600); float timeScore = (float) Math.exp(numHours); if (tags.contains(\\\"zzz\\\")) return 999999; int x = 0; x += 5; x *= 10; return timeScore + _INNER_SCORE + price;\"},\"values\":{\"srcid\":1234}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"color\",\"year\"]}}");
assertTrue(_comp.isEquals(json, expected));
}
@@ -1558,7 +1558,7 @@ public void testRelevanceModelExample1() throws Exception
"FROM cars " +
"WHERE color = 'red' " +
"USING RELEVANCE MODEL my_model ('thisYear':2001, 'goodYear':(1996)) " +
- " DEFINED AS (float _INNER_SCORE, int thisYear, int year, IntOpenHashSet goodYear) " +
+ " DEFINED AS (int thisYear, IntOpenHashSet goodYear) " +
" BEGIN " +
" if (goodYear.contains(year)) " +
" return (float)Math.exp(10d); " +
@@ -1569,7 +1569,7 @@ public void testRelevanceModelExample1() throws Exception
"ORDER BY relevance"
);
- JSONObject expected = new JSONObject("{\"sort\":\"relevance\",\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"_INNER_SCORE\",\"thisYear\",\"year\",\"goodYear\"],\"facets\":{\"int\":[\"year\"]},\"variables\":{\"set_int\":[\"goodYear\"],\"int\":[\"thisYear\"]},\"function\":\"if (goodYear.contains(year)) return (float)Math.exp(10d); if (year == thisYear) return 87f; return _INNER_SCORE;\"},\"values\":{\"thisYear\":2001,\"goodYear\":[1996]}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"*\"]}}");
+ JSONObject expected = new JSONObject("{\"sort\":\"relevance\",\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"thisYear\",\"goodYear\",\"year\",\"_INNER_SCORE\"],\"facets\":{\"int\":[\"year\"]},\"variables\":{\"set_int\":[\"goodYear\"],\"int\":[\"thisYear\"]},\"function\":\"if (goodYear.contains(year)) return (float)Math.exp(10d); if (year == thisYear) return 87f; return _INNER_SCORE;\"},\"values\":{\"thisYear\":2001,\"goodYear\":[1996]}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"*\"]}}");
assertTrue(_comp.isEquals(json, expected));
}
@@ -1584,7 +1584,7 @@ public void testRelevanceModelExample2() throws Exception
"FROM cars " +
"WHERE color = 'red' " +
"USING RELEVANCE MODEL my_model ('thisYear':2001, 'goodYear':(1996)) " +
- " DEFINED AS (float _INNER_SCORE, int thisYear, int year, String color, IntOpenHashSet goodYear) " +
+ " DEFINED AS (int thisYear, IntOpenHashSet goodYear) " +
" BEGIN " +
" if (goodYear.contains(year)) " +
" return (float)Math.exp(10d); " +
@@ -1597,7 +1597,34 @@ public void testRelevanceModelExample2() throws Exception
"ORDER BY relevance"
);
- JSONObject expected = new JSONObject("{\"sort\":\"relevance\",\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"_INNER_SCORE\",\"thisYear\",\"year\",\"color\",\"goodYear\"],\"facets\":{\"int\":[\"year\"],\"string\":[\"color\"]},\"variables\":{\"set_int\":[\"goodYear\"],\"int\":[\"thisYear\"]},\"function\":\"if (goodYear.contains(year)) return (float)Math.exp(10d); if (year == thisYear) return 87f; else if (color.equals(\\\"blue\\\")) return 99f; return _INNER_SCORE;\"},\"values\":{\"thisYear\":2001,\"goodYear\":[1996]}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"*\"]}}");
+ JSONObject expected = new JSONObject("{\"sort\":\"relevance\",\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"thisYear\",\"goodYear\",\"color\",\"year\",\"_INNER_SCORE\"],\"facets\":{\"int\":[\"year\"],\"string\":[\"color\"]},\"variables\":{\"set_int\":[\"goodYear\"],\"int\":[\"thisYear\"]},\"function\":\"if (goodYear.contains(year)) return (float)Math.exp(10d); if (year == thisYear) return 87f; else if (color.equals(\\\"blue\\\")) return 99f; return _INNER_SCORE;\"},\"values\":{\"thisYear\":2001,\"goodYear\":[1996]}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"*\"]}}");
+ assertTrue(_comp.isEquals(json, expected));
+ }
+
+ @Test
+ public void testRelevanceModelVariableScopes() throws Exception
+ {
+ System.out.println("testRelevanceModelVariableScopes");
+ System.out.println("==================================================");
+
+ JSONObject json = _compiler.compile(
+ "SELECT * " +
+ "FROM cars " +
+ "WHERE color = 'red' " +
+ "USING RELEVANCE MODEL my_model ('boost':2.5) " +
+ " DEFINED AS (float boost) " +
+ " BEGIN " +
+ " int x, y; " +
+ " for (int i = 0; i < 10; ++i) { " +
+ " x = 10; " +
+ " y = x + i; " +
+ " } " +
+ " return y * boost + price; " +
+ " END " +
+ "ORDER BY relevance"
+ );
+
+ JSONObject expected = new JSONObject("{\"sort\":\"relevance\",\"query\":{\"query_string\":{\"query\":\"\",\"relevance\":{\"model\":{\"function_params\":[\"boost\",\"price\"],\"facets\":{\"float\":[\"price\"]},\"variables\":{\"float\":[\"boost\"]},\"function\":\"int x, y; for (int i = 0; i < 10; ++i) { x = 10; y = x + i; } return y * boost + price;\"},\"values\":{\"boost\":2.5}}}},\"selections\":[{\"term\":{\"color\":{\"value\":\"red\"}}}],\"meta\":{\"select_list\":[\"*\"]}}");
assertTrue(_comp.isEquals(json, expected));
}
View
323 sensei-core/src/test/java/com/senseidb/test/bql/parsers/TestErrorHandling.java
@@ -953,4 +953,327 @@ public void testUsingRelevanceOnce() throws Exception
}
}
+ @Test
+ public void testRelevanceVarRedefined() throws Exception
+ {
+ System.out.println("testRelevanceVarRedefined");
+ System.out.println("==================================================");
+
+ boolean caughtException = false;
+ try
+ {
+ JSONObject json = _compiler.compile(
+ "SELECT * \n" +
+ "FROM cars \n" +
+ "USING RELEVANCE MODEL md1 ('srcid':1234) \n" +
+ " DEFINED AS (int srcid) \n" +
+ " BEGIN \n" +
+ " int x, y; \n" +
+ " short x = 5; \n" +
+ " return 0.5; \n" +
+ " END"
+ );
+ }
+ catch (RecognitionException err)
+ {
+ assertEquals("[line:7, col:15] Variable \"x\" is already defined. (token=;)",
+ _compiler.getErrorMessage(err));
+ caughtException = true;
+ }
+ finally
+ {
+ assertTrue(caughtException);
+ }
+ }
+
+ @Test
+ public void testRelevanceUndefinedVar1() throws Exception
+ {
+ System.out.println("testRelevanceUndefinedVar1");
+ System.out.println("==================================================");
+
+ boolean caughtException = false;
+ try
+ {
+ JSONObject json = _compiler.compile(
+ "SELECT * \n" +
+ "FROM cars \n" +
+ "USING RELEVANCE MODEL md1 ('srcid':1234) \n" +
+ " DEFINED AS (int srcid) \n" +
+ " BEGIN \n" +
+ " if (x == 5) \n" +
+ " return 0.1; \n" +
+ " return 0.5; \n" +
+ " END"
+ );
+ }
+ catch (RecognitionException err)
+ {
+ assertEquals("[line:6, col:10] Variable or class \"x\" is not defined. (token===)", _compiler.getErrorMessage(err));
+ caughtException = true;
+ }
+ finally
+ {
+ assertTrue(caughtException);
+ }
+ }
+
+ @Test
+ public void testRelevanceUndefinedVar2() throws Exception
+ {
+ System.out.println("testRelevanceUndefinedVar2");
+ System.out.println("==================================================");
+
+ boolean caughtException = false;
+ try
+ {
+ JSONObject json = _compiler.compile(
+ "SELECT * \n" +
+ "FROM cars \n" +
+ "USING RELEVANCE MODEL md1 ('srcid':1234) \n" +
+ " DEFINED AS (int srcid) \n" +
+ " BEGIN \n" +
+ " int x = 5; \n" +
+ " if (price > 2000.0) \n" +
+ " return 0.1; \n" +
+ " else { \n" +
+ " x = 10; \n" +
+ " y = x + 123; \n" +
+ " } \n" +
+ " return 0.5; \n" +
+ " END"
+ );
+ }
+ catch (RecognitionException err)
+ {
+ // System.out.println(">>> err = " + _compiler.getErrorMessage(err));
+ assertEquals("[line:11, col:8] Variable or class \"y\" is not defined. (token==)", _compiler.getErrorMessage(err));
+ caughtException = true;
+ }
+ finally
+ {
+ assertTrue(caughtException);
+ }
+ }
+
+ @Test
+ public void testRelevanceUndefinedVar3() throws Exception
+ {
+ System.out.println("testRelevanceUndefinedVar3");
+ System.out.println("==================================================");
+
+ boolean caughtException = false;
+ try
+ {
+ JSONObject json = _compiler.compile(
+ "SELECT * \n" +
+ "FROM cars \n" +
+ "USING RELEVANCE MODEL md1 ('srcid':1234) \n" +
+ " DEFINED AS (int srcid) \n" +
+ " BEGIN \n" +
+ " int total = 0; \n" +
+ " for (int i = 0; i < 10; ++i) { \n" +
+ " total += i; \n" +
+ " } \n" +
+ " i = 100; \n" +
+ " END"
+ );
+ }
+ catch (RecognitionException err)
+ {
+ assertEquals("[line:10, col:6] Variable or class \"i\" is not defined. (token==)", _compiler.getErrorMessage(err));
+ caughtException = true;
+ }
+ finally
+ {
+ assertTrue(caughtException);
+ }
+ }
+
+ @Test
+ public void testRelevanceVarDeclError1() throws Exception
+ {
+ System.out.println("testRelevanceVarDeclError1");
+ System.out.println("==================================================");
+
+ boolean caughtException = false;
+ try
+ {
+ JSONObject json = _compiler.compile(
+ "SELECT * \n" +
+ "FROM cars \n" +
+ "USING RELEVANCE MODEL md1 ('srcid':1234) \n" +
+ " DEFINED AS (int srcid) \n" +
+ " BEGIN \n" +
+ " int year; \n" +
+ " END"
+ );
+ }
+ catch (RecognitionException err)
+ {
+ assertEquals("[line:6, col:12] Facet name \"year\" cannot be used to declare a variable. (token=;)", _compiler.getErrorMessage(err));
+ caughtException = true;
+ }
+ finally
+ {
+ assertTrue(caughtException);
+ }
+ }
+
+ @Test
+ public void testRelevanceVarDeclError2() throws Exception
+ {
+ System.out.println("testRelevanceVarDeclError2");
+ System.out.println("==================================================");
+
+ boolean caughtException = false;
+ try
+ {
+ JSONObject json = _compiler.compile(
+ "SELECT * \n" +
+ "FROM cars \n" +
+ "USING RELEVANCE MODEL md1 ('srcid':1234) \n" +
+ " DEFINED AS (int srcid) \n" +
+ " BEGIN \n" +
+ " String _NOW; \n" +
+ " END"
+ );
+ }
+ catch (RecognitionException err)
+ {
+ assertEquals("[line:6, col:15] Internal variable \"_NOW\" cannot be re-used to declare another variable. (token=;)", _compiler.getErrorMessage(err));
+ caughtException = true;
+ }
+ finally
+ {
+ assertTrue(caughtException);
+ }
+ }
+
+ @Test
+ public void testRelevanceVarDeclError3() throws Exception
+ {
+ System.out.println("testRelevanceVarDeclError3");
+ System.out.println("==================================================");
+
+ boolean caughtException = false;
+ try
+ {
+ JSONObject json = _compiler.compile(
+ "SELECT * \n" +
+ "FROM cars \n" +
+ "USING RELEVANCE MODEL md1 ('srcid':1234) \n" +
+ " DEFINED AS (int srcid) \n" +
+ " BEGIN \n" +
+ " int x = 100; \n" +
+ " for (int i = 1; i < 10; ++i) { \n" +
+ " int x; \n" +
+ " } \n" +
+ " END"
+ );
+ }
+ catch (RecognitionException err)
+ {
+ assertEquals("[line:8, col:11] Variable \"x\" is already defined. (token=;)", _compiler.getErrorMessage(err));
+ caughtException = true;
+ }
+ finally
+ {
+ assertTrue(caughtException);
+ }
+ }
+
+ @Test
+ public void testRelevanceModelParamError1() throws Exception
+ {
+ System.out.println("testRelevanceModelParamError1");
+ System.out.println("==================================================");
+
+ boolean caughtException = false;
+ try
+ {
+ JSONObject json = _compiler.compile(
+ "SELECT * \n" +
+ "FROM cars \n" +
+ "USING RELEVANCE MODEL md1 ('srcid':1234) \n" +
+ " DEFINED AS (int srcid, float price) \n" +
+ " BEGIN \n" +
+ " return 0.5; \n" +
+ " END"
+ );
+ }
+ catch (RecognitionException err)
+ {
+ // System.out.println(">>> err = " + _compiler.getErrorMessage(err));
+ assertEquals("[line:4, col:36] Facet name \"price\" cannot be used as a relevance model parameter. (token=))", _compiler.getErrorMessage(err));
+ caughtException = true;
+ }
+ finally
+ {
+ assertTrue(caughtException);
+ }
+ }
+
+ @Test
+ public void testRelevanceModelParamError2() throws Exception
+ {
+ System.out.println("testRelevanceModelParamError2");
+ System.out.println("==================================================");
+
+ boolean caughtException = false;
+ try
+ {
+ JSONObject json = _compiler.compile(
+ "SELECT * \n" +
+ "FROM cars \n" +
+ "USING RELEVANCE MODEL md1 ('srcid':1234) \n" +
+ " DEFINED AS (int srcid, String srcid) \n" +
+ " BEGIN \n" +
+ " return 0.5; \n" +
+ " END"
+ );
+ }
+ catch (RecognitionException err)
+ {
+ // System.out.println(">>> err = " + _compiler.getErrorMessage(err));
+ assertEquals("[line:4, col:37] Parameter name \"srcid\" has already been used. (token=))", _compiler.getErrorMessage(err));
+ caughtException = true;
+ }
+ finally
+ {
+ assertTrue(caughtException);
+ }
+ }
+
+ @Test
+ public void testRelevanceModelParamError3() throws Exception
+ {
+ System.out.println("testRelevanceModelParamError3");
+ System.out.println("==================================================");
+
+ boolean caughtException = false;
+ try
+ {
+ JSONObject json = _compiler.compile(
+ "SELECT * \n" +
+ "FROM cars \n" +
+ "USING RELEVANCE MODEL md1 ('srcid':1234) \n" +
+ " DEFINED AS (int srcid, long _NOW) \n" +
+ " BEGIN \n" +
+ " return 0.5; \n" +
+ " END"
+ );
+ }
+ catch (RecognitionException err)
+ {
+ // System.out.println(">>> err = " + _compiler.getErrorMessage(err));
+ assertEquals("[line:4, col:34] Internal variable \"_NOW\" cannot be used as a relevance model parameter. (token=))", _compiler.getErrorMessage(err));
+ caughtException = true;
+ }
+ finally
+ {
+ assertTrue(caughtException);
+ }
+ }
+
}

0 comments on commit 2a6e942

Please sign in to comment.
Something went wrong with that request. Please try again.