[PPL] Support expression in syntax

integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/StatsCommandIT.java

@@ -15,11 +15,10 @@
 
 package com.amazon.opendistroforelasticsearch.sql.ppl;
 
-import org.junit.jupiter.api.Test;
+import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT;
 
 import java.io.IOException;
-
-import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT;
+import org.junit.jupiter.api.Test;
 
 public class StatsCommandIT extends PPLIntegTestCase {
 
@@ -81,4 +80,22 @@ public void testStatsCount() throws IOException {
   }
 
   // TODO: each stats aggregate function should be tested here when implemented
+
+  @Test
+  public void testStatsNested() throws IOException {
+    String result =
+        executeQueryToString(
+            String.format("source=%s | stats avg(abs(age)*2) as AGE", TEST_INDEX_ACCOUNT));
+    assertEquals(
+        "{\n"
+            + "  \"schema\": [{\n"
+            + "    \"name\": \"AGE\",\n"
+            + "    \"type\": \"double\"\n"
+            + "  }],\n"
+            + "  \"total\": 1,\n"
+            + "  \"datarows\": [[60.342]],\n"
+            + "  \"size\": 1\n"
+            + "}\n",
+        result);
+  }
 }

ppl/src/main/antlr/OpenDistroPPLParser.g4

@@ -72,7 +72,7 @@ sortCommand
     ;
 
 evalCommand
-    : EVAL evalExpression (COMMA evalExpression)*
+    : EVAL evalClause (COMMA evalClause)*
     ;
 
 /** clauses */
@@ -93,14 +93,18 @@ sortbyClause
     : sortField (COMMA sortField)*
     ;
 
+evalClause
+    : fieldExpression EQUAL expression
+    ;
+
 /** aggregation terms */
 statsAggTerm
     : statsFunction (AS alias=wcFieldExpression)?
     ;
 
 /** aggregation functions */
 statsFunction
-    : statsFunctionName LT_PRTHS fieldExpression RT_PRTHS           #statsFunctionCall
+    : statsFunctionName LT_PRTHS valueExpression RT_PRTHS           #statsFunctionCall
     | percentileAggFunction                                         #percentileAggFunctionCall
     ;
 
@@ -124,32 +128,32 @@ percentileAggFunction
 expression
     : logicalExpression
     | comparisonExpression
-    | evalFunctionCall
-    | binaryArithmetic
+    | valueExpression
     ;
 
 logicalExpression
     : comparisonExpression                                          #comparsion
-    | evalExpression                                                #eval
     | NOT logicalExpression                                         #logicalNot
     | left=logicalExpression OR right=logicalExpression             #logicalOr
     | left=logicalExpression (AND)? right=logicalExpression         #logicalAnd
     ;
 
-evalExpression
-    : fieldExpression EQUAL expression
+comparisonExpression
+    : left=valueExpression comparisonOperator right=valueExpression #compareExpr
+    | valueExpression IN valueList                                  #inExpr
     ;
 
-comparisonExpression
-    : left=fieldExpression comparisonOperator
-    (field=fieldExpression | literal=literalValue)                  #compareExpr
-    | fieldExpression IN valueList                                  #inExpr
+valueExpression
+    : left=valueExpression binaryOperator right=valueExpression     #binaryArithmetic
+    | LT_PRTHS left=valueExpression binaryOperator
+    right=valueExpression RT_PRTHS                                  #parentheticBinaryArithmetic
+    | primaryExpression                                             #valueExpressionDefault
     ;
 
-binaryArithmetic
-    : (leftField=fieldExpression | leftValue=literalValue) binaryOperator
-    (rightField=fieldExpression | rightValue=literalValue)
-    | LT_PRTHS binaryArithmetic RT_PRTHS
+primaryExpression
+    : evalFunctionCall
+    | fieldExpression
+    | literalValue
     ;
 
 /** tables */
@@ -186,43 +190,35 @@ wcFieldExpression
     : wcQualifiedName
     ;
 
-
 /** functions */
 evalFunctionCall
     : evalFunctionName LT_PRTHS functionArgs RT_PRTHS
     ;
 
 evalFunctionName
-    : basicFunctionNameBase | esFunctionNameBase
+    : mathematicalFunctionBase
+    | dateAndTimeFunctionBase
+    | textFunctionBase
     ;
 
-basicFunctionNameBase
-    : ABS | ACOS | ADD | ASCII | ASIN | ATAN | ATAN2 | CBRT | CEIL | CONCAT | CONCAT_WS
-    | COS | COSH | COT | CURDATE | DATE | DATE_FORMAT | DAYOFMONTH | DEGREES
-    | E | EXP | EXPM1 | FLOOR | IF | IFNULL | ISNULL | LEFT | LENGTH | LN | LOCATE | LOG
-    | LOG10 | LOG2 | LOWER | LTRIM | MAKETIME | MODULUS | MONTH | MONTHNAME | MULTIPLY
-    | NOW | PI | POW | POWER | RADIANS | RAND | REPLACE | RIGHT | RINT | ROUND | RTRIM
-    | SIGN | SIGNUM | SIN | SINH | SQRT | SUBSTRING | SUBTRACT | TAN | TIMESTAMP | TRIM
-    | UPPER | YEAR | ADDDATE | ADDTIME | GREATEST | LEAST
+functionArgs
+    : functionArg (COMMA functionArg)*
     ;
 
-esFunctionNameBase
-    : DATE_HISTOGRAM | DAY_OF_MONTH | DAY_OF_YEAR | DAY_OF_WEEK | EXCLUDE
-    | EXTENDED_STATS | FILTER | GEO_BOUNDING_BOX | GEO_CELL | GEO_DISTANCE | GEO_DISTANCE_RANGE | GEO_INTERSECTS
-    | GEO_POLYGON | INCLUDE | IN_TERMS | HISTOGRAM | HOUR_OF_DAY
-    | MATCHPHRASE | MATCH_PHRASE | MATCHQUERY | MATCH_QUERY | MINUTE_OF_DAY
-    | MINUTE_OF_HOUR | MISSING | MONTH_OF_YEAR | MULTIMATCH | MULTI_MATCH | NESTED
-    | PERCENTILES | QUERY | RANGE | REGEXP_QUERY | REVERSE_NESTED | SCORE
-    | SECOND_OF_MINUTE | STATS | TERM | TERMS | TOPHITS
-    | WEEK_OF_YEAR | WILDCARDQUERY | WILDCARD_QUERY
+functionArg
+    : valueExpression
     ;
 
-functionArgs
-    : functionArg (COMMA functionArg)*
+mathematicalFunctionBase
+    : ABS
     ;
 
-functionArg
-    : expression | fieldExpression | literalValue
+dateAndTimeFunctionBase
+    :
+    ;
+
+textFunctionBase
+    :
     ;
 
 /** operators */

ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilder.java

@@ -190,7 +190,7 @@ public UnresolvedPlan visitSortCommand(SortCommandContext ctx) {
   @Override
   public UnresolvedPlan visitEvalCommand(EvalCommandContext ctx) {
     return new Eval(
-        ctx.evalExpression()
+        ctx.evalClause()
             .stream()
             .map(ct -> (Let) visitExpression(ct))
             .collect(Collectors.toList())

ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java

@@ -20,14 +20,15 @@
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.BooleanLiteralContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.CompareExprContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.DecimalLiteralContext;
-import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.EvalExpressionContext;
+import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.EvalClauseContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.EvalFunctionCallContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.FieldExpressionContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.InExprContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.IntegerLiteralContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.LogicalAndContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.LogicalNotContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.LogicalOrContext;
+import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.ParentheticBinaryArithmeticContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.PercentileAggFunctionContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.QualifiedNameContext;
 import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.SortFieldContext;
@@ -63,7 +64,15 @@
  */
 public class AstExpressionBuilder extends OpenDistroPPLParserBaseVisitor<UnresolvedExpression> {
   /**
-   * Logical expression excluding boolean, eval, comparison.
+   * Eval clause.
+   */
+  @Override
+  public UnresolvedExpression visitEvalClause(EvalClauseContext ctx) {
+    return new Let((Field) visit(ctx.fieldExpression()), visit(ctx.expression()));
+  }
+
+  /**
+   * Logical expression excluding boolean, comparison.
    */
   @Override
   public UnresolvedExpression visitLogicalNot(LogicalNotContext ctx) {
@@ -80,45 +89,42 @@ public UnresolvedExpression visitLogicalAnd(LogicalAndContext ctx) {
     return new And(visit(ctx.left), visit(ctx.right));
   }
 
-
-  /**
-   * Eval expression.
-   */
-  @Override
-  public UnresolvedExpression visitEvalExpression(EvalExpressionContext ctx) {
-    Field field = (Field) visit(ctx.fieldExpression());
-    UnresolvedExpression evalFunctionCall = visit(ctx.expression());
-    return new Let(field, evalFunctionCall);
-  }
-
   /**
    * Comparison expression.
    */
   @Override
   public UnresolvedExpression visitCompareExpr(CompareExprContext ctx) {
-    UnresolvedExpression right = ctx.field != null ? visit(ctx.field) : visit(ctx.literal);
-    return new Compare(ctx.comparisonOperator().getText(), visit(ctx.left), right);
+    return new Compare(ctx.comparisonOperator().getText(), visit(ctx.left), visit(ctx.right));
   }
 
   @Override
   public UnresolvedExpression visitInExpr(InExprContext ctx) {
     return new In(
-        visit(ctx.fieldExpression()),
+        visit(ctx.valueExpression()),
         ctx.valueList()
             .literalValue()
             .stream()
             .map(this::visitLiteralValue)
             .collect(Collectors.toList()));
   }
 
+  /**
+   * Value Expression.
+   */
   @Override
   public UnresolvedExpression visitBinaryArithmetic(BinaryArithmeticContext ctx) {
     return new Function(
         ctx.binaryOperator().getText(),
-        Arrays.asList(
-            ctx.leftField != null ? visit(ctx.leftField) : visit(ctx.leftValue),
-            ctx.rightField != null ? visit(ctx.rightField) : visit(ctx.rightValue)
-        )
+        Arrays.asList(visit(ctx.left), visit(ctx.right))
+    );
+  }
+
+  @Override
+  public UnresolvedExpression visitParentheticBinaryArithmetic(
+      ParentheticBinaryArithmeticContext ctx) {
+    return new Function(
+        ctx.binaryOperator().getText(),
+        Arrays.asList(visit(ctx.left), visit(ctx.right))
     );
   }
 
@@ -148,7 +154,7 @@ public UnresolvedExpression visitSortField(SortFieldContext ctx) {
    */
   @Override
   public UnresolvedExpression visitStatsFunctionCall(StatsFunctionCallContext ctx) {
-    return new AggregateFunction(ctx.statsFunctionName().getText(), visit(ctx.fieldExpression()));
+    return new AggregateFunction(ctx.statsFunctionName().getText(), visit(ctx.valueExpression()));
   }
 
   @Override

ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilderTest.java

@@ -199,6 +199,39 @@ public void testStatsCommandWithAlias() {
     );
   }
 
+  @Test
+  public void testStatsCommandWithNestedFunctions() {
+    assertEqual("source=t | stats sum(a+b)",
+        agg(
+            relation("t"),
+            exprList(
+                aggregate(
+                    "sum",
+                    function("+", field("a"), field("b"))
+                )),
+            emptyList(),
+            emptyList(),
+            defaultStatsArgs()
+        ));
+    assertEqual("source=t | stats sum(abs(a)/2)",
+        agg(
+            relation("t"),
+            exprList(
+                aggregate(
+                    "sum",
+                    function(
+                        "/",
+                        function("abs", field("a")),
+                        intLiteral(2)
+                    )
+                )
+            ),
+            emptyList(),
+            emptyList(),
+            defaultStatsArgs()
+        ));
+  }
+
   @Test
   public void testDedupCommand() {
     assertEqual("source=t | dedup f1, f2",

ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilderTest.java

@@ -134,6 +134,14 @@ public void testEvalBinaryOperationExpr() {
                 function("+", field("a"), field("b"))
             )
         ));
+    assertEqual("source=t | eval f=(a+b)",
+        eval(
+            relation("t"),
+            let(
+                field("f"),
+                function("+", field("a"), field("b"))
+            )
+        ));
   }
 
   @Test