From f86fec5538780e1a547dbb2f1c2f2556df41dcfe Mon Sep 17 00:00:00 2001 From: WilliamZhu Date: Thu, 25 Aug 2016 23:19:00 +0800 Subject: [PATCH] fix issuse https://github.com/NLPchina/elasticsearch-sql/issues/265#issuecomment-242388650 --- src/main/java/org/nlpcn/es4sql/Test.java | 5 +- src/main/java/org/nlpcn/es4sql/Util.java | 159 ++++++++++-------- .../org/nlpcn/es4sql/parse/FieldMaker.java | 22 +-- .../org/nlpcn/es4sql/parse/SqlParser.java | 16 +- .../org/nlpcn/es4sql/SQLFunctionsTest.java | 2 +- .../java/org/nlpcn/es4sql/SqlParserTests.java | 16 ++ 6 files changed, 126 insertions(+), 94 deletions(-) diff --git a/src/main/java/org/nlpcn/es4sql/Test.java b/src/main/java/org/nlpcn/es4sql/Test.java index 5e739c40..1a0e3976 100644 --- a/src/main/java/org/nlpcn/es4sql/Test.java +++ b/src/main/java/org/nlpcn/es4sql/Test.java @@ -75,10 +75,7 @@ public static void main(String[] args) throws Exception { "group by key "; String TEST_INDEX = "elasticsearch-sql_test_index"; - sql = "SELECT " + - " concat_ws('-',age,'-') from " + - TEST_INDEX + "/account " + - " limit 10 "; + sql = "select count(t.*) as counts,sum(t.size) from xxx/locs as t group by t.kk"; System.out.println("sql" + sql + ":\n----------\n" + sqlToEsQuery(sql)); diff --git a/src/main/java/org/nlpcn/es4sql/Util.java b/src/main/java/org/nlpcn/es4sql/Util.java index bdf95408..eaec7ad3 100644 --- a/src/main/java/org/nlpcn/es4sql/Util.java +++ b/src/main/java/org/nlpcn/es4sql/Util.java @@ -4,6 +4,7 @@ import java.util.Map; import com.alibaba.druid.sql.ast.expr.*; +import org.nlpcn.es4sql.domain.Field; import org.nlpcn.es4sql.domain.KVValue; import org.nlpcn.es4sql.exception.SqlParseException; @@ -11,91 +12,107 @@ public class Util { - public static String joiner(List lists, String oper) { - - if (lists.size() == 0) { - return null; - } - - StringBuilder sb = new StringBuilder(lists.get(0).toString()); - for (int i = 1; i < lists.size(); i++) { - sb.append(oper); - sb.append(lists.get(i).toString()); - } - - return sb.toString(); - } - - public static List> sortByMap(List> lists) { - - return lists; - } - - public static Object expr2Object(SQLExpr expr) { - Object value = null; - if (expr instanceof SQLNumericLiteralExpr) { - value = ((SQLNumericLiteralExpr) expr).getNumber(); - } else if (expr instanceof SQLCharExpr) { - value = ((SQLCharExpr) expr).getText(); - } else if (expr instanceof SQLIdentifierExpr) { - value = expr.toString(); - } else if (expr instanceof SQLPropertyExpr) { + public static String joiner(List lists, String oper) { + + if (lists.size() == 0) { + return null; + } + + StringBuilder sb = new StringBuilder(lists.get(0).toString()); + for (int i = 1; i < lists.size(); i++) { + sb.append(oper); + sb.append(lists.get(i).toString()); + } + + return sb.toString(); + } + + public static List> sortByMap(List> lists) { + + return lists; + } + + public static Object removeTableAilasFromField(Object expr, String tableAlias) { + + if (expr instanceof SQLIdentifierExpr || expr instanceof SQLPropertyExpr || expr instanceof SQLVariantRefExpr) { + String name = expr.toString().replace("`", ""); + if (tableAlias != null) { + String aliasPrefix = tableAlias + "."; + if (name.startsWith(aliasPrefix)) { + String newFieldName = name.replaceFirst(aliasPrefix, ""); + return new SQLIdentifierExpr(newFieldName); + } + } + } + return expr; + } + + + public static Object expr2Object(SQLExpr expr) { + Object value = null; + if (expr instanceof SQLNumericLiteralExpr) { + value = ((SQLNumericLiteralExpr) expr).getNumber(); + } else if (expr instanceof SQLCharExpr) { + value = ((SQLCharExpr) expr).getText(); + } else if (expr instanceof SQLIdentifierExpr) { + value = expr.toString(); + } else if (expr instanceof SQLPropertyExpr) { value = expr.toString(); - }else if (expr instanceof SQLVariantRefExpr ){ + } else if (expr instanceof SQLVariantRefExpr) { value = expr.toString(); - }else if (expr instanceof SQLAllColumnExpr) { - value = "*"; - } else if (expr instanceof SQLValuableExpr){ - value = ((SQLValuableExpr)expr).getValue(); + } else if (expr instanceof SQLAllColumnExpr) { + value = "*"; + } else if (expr instanceof SQLValuableExpr) { + value = ((SQLValuableExpr) expr).getValue(); } else { - //throw new SqlParseException("can not support this type " + expr.getClass()); - } - return value; - } - - public static Object getScriptValue(SQLExpr expr) throws SqlParseException { - if (expr instanceof SQLIdentifierExpr || expr instanceof SQLPropertyExpr || expr instanceof SQLVariantRefExpr) { - return "doc['" + expr.toString() + "'].value"; - } else if (expr instanceof SQLValuableExpr) { - return ((SQLValuableExpr) expr).getValue(); - } - throw new SqlParseException("could not parse sqlBinaryOpExpr need to be identifier/valuable got" + expr.getClass().toString() + " with value:" + expr.toString()); - } - - public static double[] String2DoubleArr(String paramer) { - String[] split = paramer.split(","); - double[] ds = new double[split.length]; - for (int i = 0; i < ds.length; i++) { - ds[i] = Double.parseDouble(split[i].trim()); - } - return ds; - } - - public static double[] KV2DoubleArr(List params) { - double[] ds = new double[params.size()]; - int i = 0; - for (KVValue v : params) { - ds[i] = ((Number) v.value).doubleValue(); - i++; - } - return ds; - } + //throw new SqlParseException("can not support this type " + expr.getClass()); + } + return value; + } + + public static Object getScriptValue(SQLExpr expr) throws SqlParseException { + if (expr instanceof SQLIdentifierExpr || expr instanceof SQLPropertyExpr || expr instanceof SQLVariantRefExpr) { + return "doc['" + expr.toString() + "'].value"; + } else if (expr instanceof SQLValuableExpr) { + return ((SQLValuableExpr) expr).getValue(); + } + throw new SqlParseException("could not parse sqlBinaryOpExpr need to be identifier/valuable got" + expr.getClass().toString() + " with value:" + expr.toString()); + } + + public static double[] String2DoubleArr(String paramer) { + String[] split = paramer.split(","); + double[] ds = new double[split.length]; + for (int i = 0; i < ds.length; i++) { + ds[i] = Double.parseDouble(split[i].trim()); + } + return ds; + } + + public static double[] KV2DoubleArr(List params) { + double[] ds = new double[params.size()]; + int i = 0; + for (KVValue v : params) { + ds[i] = ((Number) v.value).doubleValue(); + i++; + } + return ds; + } public static String extendedToString(SQLExpr sqlExpr) { - if(sqlExpr instanceof SQLTextLiteralExpr){ + if (sqlExpr instanceof SQLTextLiteralExpr) { return ((SQLTextLiteralExpr) sqlExpr).getText(); } return sqlExpr.toString(); } - public static String[] concatStringsArrays(String[] a1,String[] a2){ + public static String[] concatStringsArrays(String[] a1, String[] a2) { String[] strings = new String[a1.length + a2.length]; - for(int i=0;i params = new ArrayList<>(); String scriptFieldAlias; @@ -86,7 +86,7 @@ private static Field makeScriptMethodField(SQLBinaryOpExpr binaryExpr, String al params.add(new SQLCharExpr(script)); - return makeMethodField("script", params, null, null, false); + return makeMethodField("script", params, null, null, tableAlias,false); } @@ -205,12 +205,12 @@ private static Field handleIdentifier(SQLExpr expr, String alias, String tableAl List paramers = Lists.newArrayList(); paramers.add(new SQLCharExpr(alias)); paramers.add(new SQLCharExpr("doc['" + newFieldName + "'].value")); - field = makeMethodField("script", paramers, null, alias, true); + field = makeMethodField("script", paramers, null, alias, tableAlias,true); } return field; } - private static MethodField makeMethodField(String name, List arguments, SQLAggregateOption option, String alias, boolean first) throws SqlParseException { + private static MethodField makeMethodField(String name, List arguments, SQLAggregateOption option, String alias,String tableAlias, boolean first) throws SqlParseException { List paramers = new LinkedList<>(); String finalMethodName = name; @@ -222,11 +222,11 @@ private static MethodField makeMethodField(String name, List arguments, if (SQLFunctions.buildInFunctions.contains(binaryOpExpr.getOperator().toString().toLowerCase())) { SQLMethodInvokeExpr mExpr = makeBinaryMethodField(binaryOpExpr, alias, first); - MethodField abc = makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), null, null, false); + MethodField abc = makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), null, null,tableAlias, false); paramers.add(new KVValue(abc.getParams().get(0).toString(), new SQLCharExpr(abc.getParams().get(1).toString()))); } else { if (!binaryOpExpr.getOperator().getName().equals("=")) { - paramers.add(new KVValue("script", makeScriptMethodField(binaryOpExpr, null))); + paramers.add(new KVValue("script", makeScriptMethodField(binaryOpExpr, null,tableAlias))); } else { SQLExpr right = binaryOpExpr.getRight(); Object value = Util.expr2Object(right); @@ -238,7 +238,7 @@ private static MethodField makeMethodField(String name, List arguments, SQLMethodInvokeExpr mExpr = (SQLMethodInvokeExpr) object; String methodName = mExpr.getMethodName().toLowerCase(); if (methodName.equals("script")) { - KVValue script = new KVValue("script", makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), null, alias, true)); + KVValue script = new KVValue("script", makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), null, alias,tableAlias, true)); paramers.add(script); } else if (methodName.equals("nested") || methodName.equals("reverse_nested")) { NestedType nestedType = new NestedType(); @@ -258,11 +258,11 @@ private static MethodField makeMethodField(String name, List arguments, paramers.add(new KVValue("children", childrenType)); } else if (SQLFunctions.buildInFunctions.contains(methodName)) { //throw new SqlParseException("only support script/nested as inner functions"); - MethodField abc = makeMethodField(methodName, mExpr.getParameters(), null, null, false); + MethodField abc = makeMethodField(methodName, mExpr.getParameters(), null, null, tableAlias,false); paramers.add(new KVValue(abc.getParams().get(0).toString(), new SQLCharExpr(abc.getParams().get(1).toString()))); } else throw new SqlParseException("only support script/nested/children as inner functions"); } else { - paramers.add(new KVValue(object)); + paramers.add(new KVValue(Util.removeTableAilasFromField(object,tableAlias))); } } diff --git a/src/main/java/org/nlpcn/es4sql/parse/SqlParser.java b/src/main/java/org/nlpcn/es4sql/parse/SqlParser.java index 60df3739..a3dfe37d 100644 --- a/src/main/java/org/nlpcn/es4sql/parse/SqlParser.java +++ b/src/main/java/org/nlpcn/es4sql/parse/SqlParser.java @@ -40,7 +40,7 @@ public Select parseSelect(SQLQueryExpr mySqlExpr) throws SqlParseException { private Select parseSelect(MySqlSelectQueryBlock query) throws SqlParseException { Select select = new Select(); - findSelect(query, select,null); + findSelect(query, select,query.getFrom().getAlias()); select.getFrom().addAll(findFrom(query.getFrom())); @@ -412,6 +412,7 @@ private void findSelect(MySqlSelectQueryBlock query, Select select,String tableA private void findGroupBy(MySqlSelectQueryBlock query, Select select) throws SqlParseException { SQLSelectGroupByClause groupBy = query.getGroupBy(); + SQLTableSource sqlTableSource = query.getFrom(); if (groupBy == null) { return; } @@ -427,31 +428,32 @@ private void findGroupBy(MySqlSelectQueryBlock query, Select select) throws SqlP if ((sqlExpr instanceof SQLParensIdentifierExpr || !(sqlExpr instanceof SQLIdentifierExpr|| sqlExpr instanceof SQLMethodInvokeExpr)) && !standardGroupBys.isEmpty()) { // flush the standard group bys - select.addGroupBy(convertExprsToFields(standardGroupBys)); + select.addGroupBy(convertExprsToFields(standardGroupBys,sqlTableSource)); standardGroupBys = new ArrayList<>(); } if (sqlExpr instanceof SQLParensIdentifierExpr) { // single item with parens (should get its own aggregation) - select.addGroupBy(FieldMaker.makeField(sqlExpr, null,null)); + select.addGroupBy(FieldMaker.makeField(sqlExpr, null,sqlTableSource.getAlias())); } else if (sqlExpr instanceof SQLListExpr) { // multiple items in their own list SQLListExpr listExpr = (SQLListExpr) sqlExpr; - select.addGroupBy(convertExprsToFields(listExpr.getItems())); + select.addGroupBy(convertExprsToFields(listExpr.getItems(),sqlTableSource)); } else { // everything else gets added to the running list of standard group bys standardGroupBys.add(sqlExpr); } } if (!standardGroupBys.isEmpty()) { - select.addGroupBy(convertExprsToFields(standardGroupBys)); + select.addGroupBy(convertExprsToFields(standardGroupBys,sqlTableSource)); } } - private List convertExprsToFields(List exprs) throws SqlParseException { + private List convertExprsToFields(List exprs,SQLTableSource sqlTableSource) throws SqlParseException { List fields = new ArrayList<>(exprs.size()); for (SQLExpr expr : exprs) { - fields.add(FieldMaker.makeField(expr, null,null)); + //here we suppose groupby field will not have alias,so set null in second parameter + fields.add(FieldMaker.makeField(expr, null,sqlTableSource.getAlias())); } return fields; } diff --git a/src/test/java/org/nlpcn/es4sql/SQLFunctionsTest.java b/src/test/java/org/nlpcn/es4sql/SQLFunctionsTest.java index 267b25e5..69599c63 100644 --- a/src/test/java/org/nlpcn/es4sql/SQLFunctionsTest.java +++ b/src/test/java/org/nlpcn/es4sql/SQLFunctionsTest.java @@ -101,7 +101,7 @@ public void concat_ws_field_and_string() throws Exception { List headers = csvResult.getHeaders(); List contents = csvResult.getLines(); String[] splits = contents.get(0).split(","); - Assert.assertTrue(splits[1].endsWith("--")); + Assert.assertTrue(splits[0].endsWith("--")); } @Test diff --git a/src/test/java/org/nlpcn/es4sql/SqlParserTests.java b/src/test/java/org/nlpcn/es4sql/SqlParserTests.java index 91624171..f8bd34d5 100644 --- a/src/test/java/org/nlpcn/es4sql/SqlParserTests.java +++ b/src/test/java/org/nlpcn/es4sql/SqlParserTests.java @@ -538,6 +538,22 @@ public void nestedFieldOnWhereNoPathComplexField() throws SqlParseException { Assert.assertEquals("message.moreNested", condition.getNestedPath()); Assert.assertEquals("message.moreNested.name", condition.getName()); } + + + @Test + public void aggFieldWithAliasTableAliasShouldBeRemoved() throws SqlParseException { + String query = "select count(t.*) as counts,sum(t.size) from xxx/locs as t group by t.kk"; + SQLExpr sqlExpr = queryToExpr(query); + Select select = parser.parseSelect((SQLQueryExpr) sqlExpr); + List fields = select.getFields(); + Assert.assertTrue(fields.size()==2); + Assert.assertEquals("COUNT(*)",fields.get(0).toString()); + Assert.assertEquals("SUM(size)",fields.get(1).toString()); + List> groups = select.getGroupBys(); + Assert.assertTrue(groups.size()==1); + Assert.assertTrue(groups.get(0).size()==1); + Assert.assertEquals("kk",groups.get(0).get(0).getName()); + } @Test public void nestedFieldOnWhereGivenPath() throws SqlParseException {