Skip to content

fix issuse https://github.com/NLPchina/elasticsearch-sql/issues/265#i… #271

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 25, 2016
Merged

fix issuse https://github.com/NLPchina/elasticsearch-sql/issues/265#i… #271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions src/main/java/org/nlpcn/es4sql/Test.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,7 @@ public static void main(String[] args) throws Exception {
"group by key ";
String TEST_INDEX = "elasticsearch-sql_test_index";

sql = "SELECT " +
" concat_ws('-',age,'-') from " +
TEST_INDEX + "/account " +
" limit 10 ";
sql = "select count(t.*) as counts,sum(t.size) from xxx/locs as t group by t.kk";

System.out.println("sql" + sql + ":\n----------\n" + sqlToEsQuery(sql));

Expand Down
159 changes: 88 additions & 71 deletions src/main/java/org/nlpcn/es4sql/Util.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,98 +4,115 @@
import java.util.Map;

import com.alibaba.druid.sql.ast.expr.*;
import org.nlpcn.es4sql.domain.Field;
import org.nlpcn.es4sql.domain.KVValue;
import org.nlpcn.es4sql.exception.SqlParseException;

import com.alibaba.druid.sql.ast.*;


public class Util {
public static String joiner(List<KVValue> lists, String oper) {

if (lists.size() == 0) {
return null;
}

StringBuilder sb = new StringBuilder(lists.get(0).toString());
for (int i = 1; i < lists.size(); i++) {
sb.append(oper);
sb.append(lists.get(i).toString());
}

return sb.toString();
}

public static List<Map<String, Object>> sortByMap(List<Map<String, Object>> lists) {

return lists;
}

public static Object expr2Object(SQLExpr expr) {
Object value = null;
if (expr instanceof SQLNumericLiteralExpr) {
value = ((SQLNumericLiteralExpr) expr).getNumber();
} else if (expr instanceof SQLCharExpr) {
value = ((SQLCharExpr) expr).getText();
} else if (expr instanceof SQLIdentifierExpr) {
value = expr.toString();
} else if (expr instanceof SQLPropertyExpr) {
public static String joiner(List<KVValue> lists, String oper) {

if (lists.size() == 0) {
return null;
}

StringBuilder sb = new StringBuilder(lists.get(0).toString());
for (int i = 1; i < lists.size(); i++) {
sb.append(oper);
sb.append(lists.get(i).toString());
}

return sb.toString();
}

public static List<Map<String, Object>> sortByMap(List<Map<String, Object>> lists) {

return lists;
}

public static Object removeTableAilasFromField(Object expr, String tableAlias) {

if (expr instanceof SQLIdentifierExpr || expr instanceof SQLPropertyExpr || expr instanceof SQLVariantRefExpr) {
String name = expr.toString().replace("`", "");
if (tableAlias != null) {
String aliasPrefix = tableAlias + ".";
if (name.startsWith(aliasPrefix)) {
String newFieldName = name.replaceFirst(aliasPrefix, "");
return new SQLIdentifierExpr(newFieldName);
}
}
}
return expr;
}


public static Object expr2Object(SQLExpr expr) {
Object value = null;
if (expr instanceof SQLNumericLiteralExpr) {
value = ((SQLNumericLiteralExpr) expr).getNumber();
} else if (expr instanceof SQLCharExpr) {
value = ((SQLCharExpr) expr).getText();
} else if (expr instanceof SQLIdentifierExpr) {
value = expr.toString();
} else if (expr instanceof SQLPropertyExpr) {
value = expr.toString();
}else if (expr instanceof SQLVariantRefExpr ){
} else if (expr instanceof SQLVariantRefExpr) {
value = expr.toString();
}else if (expr instanceof SQLAllColumnExpr) {
value = "*";
} else if (expr instanceof SQLValuableExpr){
value = ((SQLValuableExpr)expr).getValue();
} else if (expr instanceof SQLAllColumnExpr) {
value = "*";
} else if (expr instanceof SQLValuableExpr) {
value = ((SQLValuableExpr) expr).getValue();
} else {
//throw new SqlParseException("can not support this type " + expr.getClass());
}
return value;
}

public static Object getScriptValue(SQLExpr expr) throws SqlParseException {
if (expr instanceof SQLIdentifierExpr || expr instanceof SQLPropertyExpr || expr instanceof SQLVariantRefExpr) {
return "doc['" + expr.toString() + "'].value";
} else if (expr instanceof SQLValuableExpr) {
return ((SQLValuableExpr) expr).getValue();
}
throw new SqlParseException("could not parse sqlBinaryOpExpr need to be identifier/valuable got" + expr.getClass().toString() + " with value:" + expr.toString());
}

public static double[] String2DoubleArr(String paramer) {
String[] split = paramer.split(",");
double[] ds = new double[split.length];
for (int i = 0; i < ds.length; i++) {
ds[i] = Double.parseDouble(split[i].trim());
}
return ds;
}

public static double[] KV2DoubleArr(List<KVValue> params) {
double[] ds = new double[params.size()];
int i = 0;
for (KVValue v : params) {
ds[i] = ((Number) v.value).doubleValue();
i++;
}
return ds;
}
//throw new SqlParseException("can not support this type " + expr.getClass());
}
return value;
}

public static Object getScriptValue(SQLExpr expr) throws SqlParseException {
if (expr instanceof SQLIdentifierExpr || expr instanceof SQLPropertyExpr || expr instanceof SQLVariantRefExpr) {
return "doc['" + expr.toString() + "'].value";
} else if (expr instanceof SQLValuableExpr) {
return ((SQLValuableExpr) expr).getValue();
}
throw new SqlParseException("could not parse sqlBinaryOpExpr need to be identifier/valuable got" + expr.getClass().toString() + " with value:" + expr.toString());
}

public static double[] String2DoubleArr(String paramer) {
String[] split = paramer.split(",");
double[] ds = new double[split.length];
for (int i = 0; i < ds.length; i++) {
ds[i] = Double.parseDouble(split[i].trim());
}
return ds;
}

public static double[] KV2DoubleArr(List<KVValue> params) {
double[] ds = new double[params.size()];
int i = 0;
for (KVValue v : params) {
ds[i] = ((Number) v.value).doubleValue();
i++;
}
return ds;
}


public static String extendedToString(SQLExpr sqlExpr) {
if(sqlExpr instanceof SQLTextLiteralExpr){
if (sqlExpr instanceof SQLTextLiteralExpr) {
return ((SQLTextLiteralExpr) sqlExpr).getText();
}
return sqlExpr.toString();
}

public static String[] concatStringsArrays(String[] a1,String[] a2){
public static String[] concatStringsArrays(String[] a1, String[] a2) {
String[] strings = new String[a1.length + a2.length];
for(int i=0;i<a1.length;i++){
for (int i = 0; i < a1.length; i++) {
strings[i] = a1[i];
}
for(int i = 0;i<a2.length;i++){
strings[a1.length+i] = a2[i];
for (int i = 0; i < a2.length; i++) {
strings[a1.length + i] = a2[i];
}
return strings;
}
Expand Down
22 changes: 11 additions & 11 deletions src/main/java/org/nlpcn/es4sql/parse/FieldMaker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,10 @@ public static Field makeField(SQLExpr expr, String alias, String tableAlias) thr
return makeFilterMethodField(mExpr, alias);
}

return makeMethodField(methodName, mExpr.getParameters(), null, alias, true);
return makeMethodField(methodName, mExpr.getParameters(), null, alias, tableAlias,true);
} else if (expr instanceof SQLAggregateExpr) {
SQLAggregateExpr sExpr = (SQLAggregateExpr) expr;
return makeMethodField(sExpr.getMethodName(), sExpr.getArguments(), sExpr.getOption(), alias, true);
return makeMethodField(sExpr.getMethodName(), sExpr.getArguments(), sExpr.getOption(), alias,tableAlias, true);
} else {
throw new SqlParseException("unknown field name : " + expr);
}
Expand All @@ -70,7 +70,7 @@ private static Object getScriptValue(SQLExpr expr) throws SqlParseException {
return Util.getScriptValue(expr);
}

private static Field makeScriptMethodField(SQLBinaryOpExpr binaryExpr, String alias) throws SqlParseException {
private static Field makeScriptMethodField(SQLBinaryOpExpr binaryExpr, String alias,String tableAlias) throws SqlParseException {
List<SQLExpr> params = new ArrayList<>();

String scriptFieldAlias;
Expand All @@ -86,7 +86,7 @@ private static Field makeScriptMethodField(SQLBinaryOpExpr binaryExpr, String al

params.add(new SQLCharExpr(script));

return makeMethodField("script", params, null, null, false);
return makeMethodField("script", params, null, null, tableAlias,false);
}


Expand Down Expand Up @@ -205,12 +205,12 @@ private static Field handleIdentifier(SQLExpr expr, String alias, String tableAl
List<SQLExpr> paramers = Lists.newArrayList();
paramers.add(new SQLCharExpr(alias));
paramers.add(new SQLCharExpr("doc['" + newFieldName + "'].value"));
field = makeMethodField("script", paramers, null, alias, true);
field = makeMethodField("script", paramers, null, alias, tableAlias,true);
}
return field;
}

private static MethodField makeMethodField(String name, List<SQLExpr> arguments, SQLAggregateOption option, String alias, boolean first) throws SqlParseException {
private static MethodField makeMethodField(String name, List<SQLExpr> arguments, SQLAggregateOption option, String alias,String tableAlias, boolean first) throws SqlParseException {
List<KVValue> paramers = new LinkedList<>();
String finalMethodName = name;

Expand All @@ -222,11 +222,11 @@ private static MethodField makeMethodField(String name, List<SQLExpr> arguments,

if (SQLFunctions.buildInFunctions.contains(binaryOpExpr.getOperator().toString().toLowerCase())) {
SQLMethodInvokeExpr mExpr = makeBinaryMethodField(binaryOpExpr, alias, first);
MethodField abc = makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), null, null, false);
MethodField abc = makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), null, null,tableAlias, false);
paramers.add(new KVValue(abc.getParams().get(0).toString(), new SQLCharExpr(abc.getParams().get(1).toString())));
} else {
if (!binaryOpExpr.getOperator().getName().equals("=")) {
paramers.add(new KVValue("script", makeScriptMethodField(binaryOpExpr, null)));
paramers.add(new KVValue("script", makeScriptMethodField(binaryOpExpr, null,tableAlias)));
} else {
SQLExpr right = binaryOpExpr.getRight();
Object value = Util.expr2Object(right);
Expand All @@ -238,7 +238,7 @@ private static MethodField makeMethodField(String name, List<SQLExpr> arguments,
SQLMethodInvokeExpr mExpr = (SQLMethodInvokeExpr) object;
String methodName = mExpr.getMethodName().toLowerCase();
if (methodName.equals("script")) {
KVValue script = new KVValue("script", makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), null, alias, true));
KVValue script = new KVValue("script", makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), null, alias,tableAlias, true));
paramers.add(script);
} else if (methodName.equals("nested") || methodName.equals("reverse_nested")) {
NestedType nestedType = new NestedType();
Expand All @@ -258,11 +258,11 @@ private static MethodField makeMethodField(String name, List<SQLExpr> arguments,
paramers.add(new KVValue("children", childrenType));
} else if (SQLFunctions.buildInFunctions.contains(methodName)) {
//throw new SqlParseException("only support script/nested as inner functions");
MethodField abc = makeMethodField(methodName, mExpr.getParameters(), null, null, false);
MethodField abc = makeMethodField(methodName, mExpr.getParameters(), null, null, tableAlias,false);
paramers.add(new KVValue(abc.getParams().get(0).toString(), new SQLCharExpr(abc.getParams().get(1).toString())));
} else throw new SqlParseException("only support script/nested/children as inner functions");
} else {
paramers.add(new KVValue(object));
paramers.add(new KVValue(Util.removeTableAilasFromField(object,tableAlias)));
}

}
Expand Down
16 changes: 9 additions & 7 deletions src/main/java/org/nlpcn/es4sql/parse/SqlParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public Select parseSelect(SQLQueryExpr mySqlExpr) throws SqlParseException {
private Select parseSelect(MySqlSelectQueryBlock query) throws SqlParseException {
Select select = new Select();

findSelect(query, select,null);
findSelect(query, select,query.getFrom().getAlias());

select.getFrom().addAll(findFrom(query.getFrom()));

Expand Down Expand Up @@ -412,6 +412,7 @@ private void findSelect(MySqlSelectQueryBlock query, Select select,String tableA

private void findGroupBy(MySqlSelectQueryBlock query, Select select) throws SqlParseException {
SQLSelectGroupByClause groupBy = query.getGroupBy();
SQLTableSource sqlTableSource = query.getFrom();
if (groupBy == null) {
return;
}
Expand All @@ -427,31 +428,32 @@ private void findGroupBy(MySqlSelectQueryBlock query, Select select) throws SqlP

if ((sqlExpr instanceof SQLParensIdentifierExpr || !(sqlExpr instanceof SQLIdentifierExpr|| sqlExpr instanceof SQLMethodInvokeExpr)) && !standardGroupBys.isEmpty()) {
// flush the standard group bys
select.addGroupBy(convertExprsToFields(standardGroupBys));
select.addGroupBy(convertExprsToFields(standardGroupBys,sqlTableSource));
standardGroupBys = new ArrayList<>();
}

if (sqlExpr instanceof SQLParensIdentifierExpr) {
// single item with parens (should get its own aggregation)
select.addGroupBy(FieldMaker.makeField(sqlExpr, null,null));
select.addGroupBy(FieldMaker.makeField(sqlExpr, null,sqlTableSource.getAlias()));
} else if (sqlExpr instanceof SQLListExpr) {
// multiple items in their own list
SQLListExpr listExpr = (SQLListExpr) sqlExpr;
select.addGroupBy(convertExprsToFields(listExpr.getItems()));
select.addGroupBy(convertExprsToFields(listExpr.getItems(),sqlTableSource));
} else {
// everything else gets added to the running list of standard group bys
standardGroupBys.add(sqlExpr);
}
}
if (!standardGroupBys.isEmpty()) {
select.addGroupBy(convertExprsToFields(standardGroupBys));
select.addGroupBy(convertExprsToFields(standardGroupBys,sqlTableSource));
}
}

private List<Field> convertExprsToFields(List<? extends SQLExpr> exprs) throws SqlParseException {
private List<Field> convertExprsToFields(List<? extends SQLExpr> exprs,SQLTableSource sqlTableSource) throws SqlParseException {
List<Field> fields = new ArrayList<>(exprs.size());
for (SQLExpr expr : exprs) {
fields.add(FieldMaker.makeField(expr, null,null));
//here we suppose groupby field will not have alias,so set null in second parameter
fields.add(FieldMaker.makeField(expr, null,sqlTableSource.getAlias()));
}
return fields;
}
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/org/nlpcn/es4sql/SQLFunctionsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ public void concat_ws_field_and_string() throws Exception {
List<String> headers = csvResult.getHeaders();
List<String> contents = csvResult.getLines();
String[] splits = contents.get(0).split(",");
Assert.assertTrue(splits[1].endsWith("--"));
Assert.assertTrue(splits[0].endsWith("--"));
}

@Test
Expand Down
16 changes: 16 additions & 0 deletions src/test/java/org/nlpcn/es4sql/SqlParserTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -538,6 +538,22 @@ public void nestedFieldOnWhereNoPathComplexField() throws SqlParseException {
Assert.assertEquals("message.moreNested", condition.getNestedPath());
Assert.assertEquals("message.moreNested.name", condition.getName());
}


@Test
public void aggFieldWithAliasTableAliasShouldBeRemoved() throws SqlParseException {
String query = "select count(t.*) as counts,sum(t.size) from xxx/locs as t group by t.kk";
SQLExpr sqlExpr = queryToExpr(query);
Select select = parser.parseSelect((SQLQueryExpr) sqlExpr);
List<Field> fields = select.getFields();
Assert.assertTrue(fields.size()==2);
Assert.assertEquals("COUNT(*)",fields.get(0).toString());
Assert.assertEquals("SUM(size)",fields.get(1).toString());
List<List<Field>> groups = select.getGroupBys();
Assert.assertTrue(groups.size()==1);
Assert.assertTrue(groups.get(0).size()==1);
Assert.assertEquals("kk",groups.get(0).get(0).getName());
}

@Test
public void nestedFieldOnWhereGivenPath() throws SqlParseException {
Expand Down