diff --git a/pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java index 2fb53b0a6f8..3dbb3fd046e 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/request/context/RequestContextUtils.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.apache.commons.lang3.EnumUtils; import org.apache.pinot.common.request.Expression; import org.apache.pinot.common.request.ExpressionType; import org.apache.pinot.common.request.FilterOperator; @@ -37,6 +38,7 @@ import org.apache.pinot.common.request.context.predicate.TextMatchPredicate; import org.apache.pinot.common.utils.RegexpPatternConverterUtils; import org.apache.pinot.common.utils.request.FilterQueryTree; +import org.apache.pinot.common.utils.request.RequestUtils; import org.apache.pinot.pql.parsers.Pql2Compiler; import org.apache.pinot.pql.parsers.pql2.ast.AstNode; import org.apache.pinot.pql.parsers.pql2.ast.FilterKind; @@ -163,10 +165,37 @@ public static FunctionContext getFunction(FunctionCallAstNode astNode) { /** * Converts the given Thrift {@link Expression} into a {@link FilterContext}. *

NOTE: Currently the query engine only accepts string literals as the right-hand side of the predicate, so we - * always convert the right-hand side expressions into strings. + * always convert the right-hand side expressions into strings. We also update boolean predicates that are + * missing an EQUALS filter operator. */ public static FilterContext getFilter(Expression thriftExpression) { - Function thriftFunction = thriftExpression.getFunctionCall(); + ExpressionType type = thriftExpression.getType(); + switch (type) { + case FUNCTION: + Function thriftFunction = thriftExpression.getFunctionCall(); + return getFilter(thriftFunction); + case IDENTIFIER: + // Convert "WHERE a" to "WHERE a = true" + return new FilterContext(FilterContext.Type.PREDICATE, null, + new EqPredicate(getExpression(thriftExpression), getStringValue(RequestUtils.getLiteralExpression(true)))); + case LITERAL: + // TODO: Handle literals. + throw new IllegalStateException(); + default: + throw new IllegalStateException(); + } + } + + public static FilterContext getFilter(Function thriftFunction) { + String functionOperator = thriftFunction.getOperator(); + + // convert "WHERE startsWith(col, 'str')" to "WHERE startsWith(col, 'str') = true" + if (!EnumUtils.isValidEnum(FilterKind.class, functionOperator)) { + return new FilterContext(FilterContext.Type.PREDICATE, null, + new EqPredicate(ExpressionContext.forFunction(getFunction(thriftFunction)), + getStringValue(RequestUtils.getLiteralExpression(true)))); + } + FilterKind filterKind = FilterKind.valueOf(thriftFunction.getOperator().toUpperCase()); List operands = thriftFunction.getOperands(); int numOperands = operands.size(); @@ -185,7 +214,8 @@ public static FilterContext getFilter(Expression thriftExpression) { return new FilterContext(FilterContext.Type.OR, children, null); case NOT: assert numOperands == 1; - return new FilterContext(FilterContext.Type.NOT, new ArrayList<>(Collections.singletonList(getFilter(operands.get(0)))), null); + return new FilterContext(FilterContext.Type.NOT, + new ArrayList<>(Collections.singletonList(getFilter(operands.get(0)))), null); case EQUALS: return new FilterContext(FilterContext.Type.PREDICATE, null, new EqPredicate(getExpression(operands.get(0)), getStringValue(operands.get(1)))); @@ -264,10 +294,36 @@ private static String getStringValue(Expression thriftExpression) { /** * Converts the given filter {@link ExpressionContext} into a {@link FilterContext}. *

NOTE: Currently the query engine only accepts string literals as the right-hand side of the predicate, so we - * always convert the right-hand side expressions into strings. + * always convert the right-hand side expressions into strings. We also update boolean predicates that are + * missing an EQUALS filter operator. */ public static FilterContext getFilter(ExpressionContext filterExpression) { - FunctionContext filterFunction = filterExpression.getFunction(); + ExpressionContext.Type type = filterExpression.getType(); + switch (type) { + case FUNCTION: + FunctionContext filterFunction = filterExpression.getFunction(); + return getFilter(filterFunction); + case IDENTIFIER: + return new FilterContext(FilterContext.Type.PREDICATE, null, + new EqPredicate(filterExpression, getStringValue(RequestUtils.getLiteralExpression(true)))); + case LITERAL: + // TODO: Handle literals + throw new IllegalStateException(); + default: + throw new IllegalStateException(); + } + } + + public static FilterContext getFilter(FunctionContext filterFunction) { + String functionOperator = filterFunction.getFunctionName().toUpperCase(); + + // convert "WHERE startsWith(col, 'str')" to "WHERE startsWith(col, 'str') = true" + if (!EnumUtils.isValidEnum(FilterKind.class, functionOperator)) { + return new FilterContext(FilterContext.Type.PREDICATE, null, + new EqPredicate(ExpressionContext.forFunction(filterFunction), + getStringValue(RequestUtils.getLiteralExpression(true)))); + } + FilterKind filterKind = FilterKind.valueOf(filterFunction.getFunctionName().toUpperCase()); List operands = filterFunction.getArguments(); int numOperands = operands.size(); @@ -286,7 +342,8 @@ public static FilterContext getFilter(ExpressionContext filterExpression) { return new FilterContext(FilterContext.Type.OR, children, null); case NOT: assert numOperands == 1; - return new FilterContext(FilterContext.Type.NOT, new ArrayList<>(Collections.singletonList(getFilter(operands.get(0)))), null); + return new FilterContext(FilterContext.Type.NOT, + new ArrayList<>(Collections.singletonList(getFilter(operands.get(0)))), null); case EQUALS: return new FilterContext(FilterContext.Type.PREDICATE, null, new EqPredicate(operands.get(0), getStringValue(operands.get(1)))); diff --git a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java index ae5c8ae7a6b..d24945baf0d 100644 --- a/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java +++ b/pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/PredicateComparisonRewriter.java @@ -18,9 +18,12 @@ */ package org.apache.pinot.sql.parsers.rewriter; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.apache.commons.lang3.EnumUtils; import org.apache.pinot.common.request.Expression; +import org.apache.pinot.common.request.ExpressionType; import org.apache.pinot.common.request.Function; import org.apache.pinot.common.request.PinotQuery; import org.apache.pinot.common.utils.request.RequestUtils; @@ -33,34 +36,69 @@ public class PredicateComparisonRewriter implements QueryRewriter { public PinotQuery rewrite(PinotQuery pinotQuery) { Expression filterExpression = pinotQuery.getFilterExpression(); if (filterExpression != null) { - pinotQuery.setFilterExpression(updateComparisonPredicate(filterExpression)); + pinotQuery.setFilterExpression(updatePredicate(filterExpression)); } Expression havingExpression = pinotQuery.getHavingExpression(); if (havingExpression != null) { - pinotQuery.setHavingExpression(updateComparisonPredicate(havingExpression)); + pinotQuery.setHavingExpression(updatePredicate(havingExpression)); } return pinotQuery; } - // This method converts a predicate expression to the what Pinot could evaluate. - // For comparison expression, left operand could be any expression, but right operand only - // supports literal. - // E.g. 'WHERE a > b' will be updated to 'WHERE a - b > 0' - private static Expression updateComparisonPredicate(Expression expression) { + /** + * This method converts an expression to what Pinot could evaluate. + * 1. For comparison expression, left operand could be any expression, but right operand only + * supports literal. E.g. 'WHERE a > b' will be converted to 'WHERE a - b > 0' + * 2. Updates boolean predicates (literals and scalar functions) that are missing an EQUALS filter. + * E.g. 1: 'WHERE a' will be updated to 'WHERE a = true' + * E.g. 2: "WHERE startsWith(col, 'str')" will be updated to "WHERE startsWith(col, 'str') = true" + * + * @param expression current expression in the expression tree + * @return re-written expression. + */ + private static Expression updatePredicate(Expression expression) { + ExpressionType type = expression.getType(); + + switch (type) { + case FUNCTION: + return updateFunctionExpression(expression); + case IDENTIFIER: + return convertPredicateToEqualsBooleanExpression(expression); + case LITERAL: + // TODO: Convert literals to boolean expressions + return expression; + default: + throw new IllegalStateException(); + } + } + + /** + * Rewrites a function expression. + * + * @param expression + * @return re-written expression + */ + private static Expression updateFunctionExpression(Expression expression) { Function function = expression.getFunctionCall(); - if (function != null) { - FilterKind filterKind; - try { - filterKind = FilterKind.valueOf(function.getOperator()); - } catch (Exception e) { - throw new SqlCompilationException("Unsupported filter kind: " + function.getOperator()); - } + String functionOperator = function.getOperator(); + + if (!EnumUtils.isValidEnum(FilterKind.class, functionOperator)) { + // If the function is not of FilterKind, we have to rewrite the function. + // Example: A query like "select col1 from table where startsWith(col1, 'myStr') AND col2 > 10;" should be + // rewritten to "select col1 from table where startsWith(col1, 'myStr') = true AND col2 > 10;". + expression = convertPredicateToEqualsBooleanExpression(expression); + return expression; + } else { + FilterKind filterKind = FilterKind.valueOf(function.getOperator()); List operands = function.getOperands(); switch (filterKind) { case AND: case OR: case NOT: - operands.replaceAll(PredicateComparisonRewriter::updateComparisonPredicate); + for (int i = 0; i < operands.size(); i++) { + Expression operand = operands.get(i); + operands.set(i, updatePredicate(operand)); + } break; case EQUALS: case NOT_EQUALS: @@ -102,9 +140,30 @@ private static Expression updateComparisonPredicate(Expression expression) { break; } } + return expression; } + /** + * Rewrite predicates to boolean expressions with EQUALS operator + * Example1: "select * from table where col1" converts to + * "select * from table where col1 = true" + * Example2: "select * from table where startsWith(col1, 'str')" converts to + * "select * from table where startsWith(col1, 'str') = true" + * @param expression Expression + * @return Rewritten expression + */ + private static Expression convertPredicateToEqualsBooleanExpression(Expression expression) { + Expression newExpression; + newExpression = RequestUtils.getFunctionExpression(FilterKind.EQUALS.name()); + List operands = new ArrayList<>(); + operands.add(expression); + operands.add(RequestUtils.getLiteralExpression(true)); + newExpression.getFunctionCall().setOperands(operands); + + return newExpression; + } + /** * The purpose of this method is to convert expression "0 < columnA" to "columnA > 0". * The conversion would be: diff --git a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java index 0bb6035986e..b206951c001 100644 --- a/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/sql/parsers/CalciteSqlCompilerTest.java @@ -194,45 +194,148 @@ public void testQuotedStrings() { @Test public void testFilterClauses() { - PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetables where a > 1.5"); - Function func = pinotQuery.getFilterExpression().getFunctionCall(); - Assert.assertEquals(func.getOperator(), FilterKind.GREATER_THAN.name()); - Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(), "a"); - Assert.assertEquals(func.getOperands().get(1).getLiteral().getDoubleValue(), 1.5); - pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetables where b < 100"); - func = pinotQuery.getFilterExpression().getFunctionCall(); - Assert.assertEquals(func.getOperator(), FilterKind.LESS_THAN.name()); - Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(), "b"); - Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(), 100L); - pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetables where c >= 10"); - func = pinotQuery.getFilterExpression().getFunctionCall(); - Assert.assertEquals(func.getOperator(), FilterKind.GREATER_THAN_OR_EQUAL.name()); - Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(), "c"); - Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(), 10L); - pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetables where d <= 50"); - func = pinotQuery.getFilterExpression().getFunctionCall(); - Assert.assertEquals(func.getOperator(), FilterKind.LESS_THAN_OR_EQUAL.name()); - Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(), "d"); - Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(), 50L); - pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetables where e BETWEEN 70 AND 80"); - func = pinotQuery.getFilterExpression().getFunctionCall(); - Assert.assertEquals(func.getOperator(), FilterKind.BETWEEN.name()); - Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(), "e"); - Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(), 70L); - Assert.assertEquals(func.getOperands().get(2).getLiteral().getLongValue(), 80L); - pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetables where regexp_like(E, '^U.*')"); - func = pinotQuery.getFilterExpression().getFunctionCall(); - Assert.assertEquals(func.getOperator(), "REGEXP_LIKE"); - Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(), "E"); - Assert.assertEquals(func.getOperands().get(1).getLiteral().getStringValue(), "^U.*"); - pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetables where g IN (12, 13, 15.2, 17)"); - func = pinotQuery.getFilterExpression().getFunctionCall(); - Assert.assertEquals(func.getOperator(), FilterKind.IN.name()); - Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(), "g"); - Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(), 12L); - Assert.assertEquals(func.getOperands().get(2).getLiteral().getLongValue(), 13L); - Assert.assertEquals(func.getOperands().get(3).getLiteral().getDoubleValue(), 15.2); - Assert.assertEquals(func.getOperands().get(4).getLiteral().getLongValue(), 17L); + { + PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetables where a > 1.5"); + Function func = pinotQuery.getFilterExpression().getFunctionCall(); + Assert.assertEquals(func.getOperator(), FilterKind.GREATER_THAN.name()); + Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(), "a"); + Assert.assertEquals(func.getOperands().get(1).getLiteral().getDoubleValue(), 1.5); + } + + { + PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetables where b < 100"); + Function func = pinotQuery.getFilterExpression().getFunctionCall(); + Assert.assertEquals(func.getOperator(), FilterKind.LESS_THAN.name()); + Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(), "b"); + Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(), 100L); + } + + { + PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetables where c >= 10"); + Function func = pinotQuery.getFilterExpression().getFunctionCall(); + Assert.assertEquals(func.getOperator(), FilterKind.GREATER_THAN_OR_EQUAL.name()); + Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(), "c"); + Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(), 10L); + } + + { + PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetables where d <= 50"); + Function func = pinotQuery.getFilterExpression().getFunctionCall(); + Assert.assertEquals(func.getOperator(), FilterKind.LESS_THAN_OR_EQUAL.name()); + Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(), "d"); + Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(), 50L); + } + + { + PinotQuery pinotQuery = + CalciteSqlParser.compileToPinotQuery("select * from vegetables where e BETWEEN 70 AND 80"); + Function func = pinotQuery.getFilterExpression().getFunctionCall(); + Assert.assertEquals(func.getOperator(), FilterKind.BETWEEN.name()); + Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(), "e"); + Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(), 70L); + Assert.assertEquals(func.getOperands().get(2).getLiteral().getLongValue(), 80L); + } + + { + PinotQuery pinotQuery = + CalciteSqlParser.compileToPinotQuery("select * from vegetables where regexp_like(E, '^U.*')"); + Function func = pinotQuery.getFilterExpression().getFunctionCall(); + Assert.assertEquals(func.getOperator(), "REGEXP_LIKE"); + Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(), "E"); + Assert.assertEquals(func.getOperands().get(1).getLiteral().getStringValue(), "^U.*"); + } + + { + PinotQuery pinotQuery = + CalciteSqlParser.compileToPinotQuery("select * from vegetables where g IN (12, 13, 15.2, 17)"); + Function func = pinotQuery.getFilterExpression().getFunctionCall(); + Assert.assertEquals(func.getOperator(), FilterKind.IN.name()); + Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(), "g"); + Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(), 12L); + Assert.assertEquals(func.getOperands().get(2).getLiteral().getLongValue(), 13L); + Assert.assertEquals(func.getOperands().get(3).getLiteral().getDoubleValue(), 15.2); + Assert.assertEquals(func.getOperands().get(4).getLiteral().getLongValue(), 17L); + } + + { + PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetable where g"); + Function func = pinotQuery.getFilterExpression().getFunctionCall(); + Assert.assertEquals(func.getOperator(), FilterKind.EQUALS.name()); + Assert.assertEquals(func.getOperands().get(0).getIdentifier().getName(), "g"); + Assert.assertEquals(func.getOperands().get(1).getLiteral(), Literal.boolValue(true)); + } + + { + PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetable where g or f = true"); + Function func = pinotQuery.getFilterExpression().getFunctionCall(); + Assert.assertEquals(func.getOperator(), FilterKind.OR.name()); + List operands = func.getOperands(); + Assert.assertEquals(operands.size(), 2); + Assert.assertEquals(operands.get(0).getFunctionCall().getOperator(), FilterKind.EQUALS.name()); + List eqOperands = operands.get(0).getFunctionCall().getOperands(); + Assert.assertEquals(eqOperands.get(0).getIdentifier().getName(), "g"); + Assert.assertEquals(eqOperands.get(1).getLiteral(), Literal.boolValue(true)); + eqOperands = operands.get(1).getFunctionCall().getOperands(); + Assert.assertEquals(eqOperands.get(0).getIdentifier().getName(), "f"); + Assert.assertEquals(eqOperands.get(1).getLiteral(), Literal.stringValue("true")); + } + + { + PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetable where startsWith(g, " + + "'str')"); + Function func = pinotQuery.getFilterExpression().getFunctionCall(); + Assert.assertEquals(func.getOperator(), FilterKind.EQUALS.name()); + Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperator(), "startswith"); + Assert.assertEquals(func.getOperands().get(1).getLiteral(), Literal.boolValue(true)); + } + + { + PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetable where startsWith(g, " + + "'str')=true and startsWith(f, 'str')"); + Function func = pinotQuery.getFilterExpression().getFunctionCall(); + Assert.assertEquals(func.getOperator(), FilterKind.AND.name()); + List operands = func.getOperands(); + Assert.assertEquals(operands.size(), 2); + + Assert.assertEquals(operands.get(0).getFunctionCall().getOperator(), FilterKind.EQUALS.name()); + List eqOperands = operands.get(0).getFunctionCall().getOperands(); + Assert.assertEquals(eqOperands.get(0).getFunctionCall().getOperator(), "startswith"); + Assert.assertEquals(eqOperands.get(1).getLiteral(), Literal.stringValue("true")); + + Assert.assertEquals(operands.get(1).getFunctionCall().getOperator(), FilterKind.EQUALS.name()); + eqOperands = operands.get(1).getFunctionCall().getOperands(); + Assert.assertEquals(eqOperands.get(0).getFunctionCall().getOperator(), "startswith"); + Assert.assertEquals(eqOperands.get(1).getLiteral(), Literal.boolValue(true)); + } + + { + PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetable where (startsWith(g, " + + "'str')=true and startsWith(f, 'str')) AND (e and d=true)"); + Function func = pinotQuery.getFilterExpression().getFunctionCall(); + Assert.assertEquals(func.getOperator(), FilterKind.AND.name()); + List operands = func.getOperands(); + Assert.assertEquals(operands.size(), 4); + + Assert.assertEquals(operands.get(0).getFunctionCall().getOperator(), FilterKind.EQUALS.name()); + List eqOperands = operands.get(0).getFunctionCall().getOperands(); + Assert.assertEquals(eqOperands.get(0).getFunctionCall().getOperator(), "startswith"); + Assert.assertEquals(eqOperands.get(1).getLiteral(), Literal.stringValue("true")); + + Assert.assertEquals(operands.get(1).getFunctionCall().getOperator(), FilterKind.EQUALS.name()); + eqOperands = operands.get(1).getFunctionCall().getOperands(); + Assert.assertEquals(eqOperands.get(0).getFunctionCall().getOperator(), "startswith"); + Assert.assertEquals(eqOperands.get(1).getLiteral(), Literal.boolValue(true)); + + Assert.assertEquals(operands.get(2).getFunctionCall().getOperator(), FilterKind.EQUALS.name()); + eqOperands = operands.get(2).getFunctionCall().getOperands(); + Assert.assertEquals(eqOperands.get(0).getIdentifier().getName(), "e"); + Assert.assertEquals(eqOperands.get(1).getLiteral(), Literal.boolValue(true)); + + Assert.assertEquals(operands.get(3).getFunctionCall().getOperator(), FilterKind.EQUALS.name()); + eqOperands = operands.get(3).getFunctionCall().getOperands(); + Assert.assertEquals(eqOperands.get(0).getIdentifier().getName(), "d"); + Assert.assertEquals(eqOperands.get(1).getLiteral(), Literal.stringValue("true")); + } } @Test @@ -241,29 +344,30 @@ public void testFilterClausesWithRightExpression() { Function func = pinotQuery.getFilterExpression().getFunctionCall(); Assert.assertEquals(func.getOperator(), FilterKind.GREATER_THAN.name()); Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperator(), "minus"); - Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(), - "a"); - Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(1).getIdentifier().getName(), - "b"); + Assert + .assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(), "a"); + Assert + .assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(1).getIdentifier().getName(), "b"); Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(), 0L); pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetables where 0 < a-b"); func = pinotQuery.getFilterExpression().getFunctionCall(); Assert.assertEquals(func.getOperator(), FilterKind.GREATER_THAN.name()); Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperator(), "minus"); - Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(), - "a"); - Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(1).getIdentifier().getName(), - "b"); + Assert + .assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(), "a"); + Assert + .assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(1).getIdentifier().getName(), "b"); Assert.assertEquals(func.getOperands().get(1).getLiteral().getLongValue(), 0L); pinotQuery = CalciteSqlParser.compileToPinotQuery("select * from vegetables where b < 100 + c"); func = pinotQuery.getFilterExpression().getFunctionCall(); Assert.assertEquals(func.getOperator(), FilterKind.LESS_THAN.name()); Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperator(), "minus"); - Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(), - "b"); - Assert.assertEquals( - func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperator(), "plus"); + Assert + .assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(), "b"); + Assert + .assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperator(), + "plus"); Assert.assertEquals( func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperands().get(0) .getLiteral().getLongValue(), 100L); @@ -275,10 +379,11 @@ public void testFilterClausesWithRightExpression() { func = pinotQuery.getFilterExpression().getFunctionCall(); Assert.assertEquals(func.getOperator(), FilterKind.LESS_THAN.name()); Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperator(), "minus"); - Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(), - "b"); - Assert.assertEquals( - func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperator(), "plus"); + Assert + .assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(), "b"); + Assert + .assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperator(), + "plus"); Assert.assertEquals( func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperands().get(0) .getLiteral().getLongValue(), 100L); @@ -292,10 +397,12 @@ public void testFilterClausesWithRightExpression() { func = pinotQuery.getFilterExpression().getFunctionCall(); Assert.assertEquals(func.getOperator(), FilterKind.LESS_THAN_OR_EQUAL.name()); Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperator(), "minus"); - Assert.assertEquals( - func.getOperands().get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperator(), "foo1"); - Assert.assertEquals( - func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperator(), "foo2"); + Assert + .assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperator(), + "foo1"); + Assert + .assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperator(), + "foo2"); Assert.assertEquals( func.getOperands().get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperands().get(0) .getFunctionCall().getOperator(), "bar1"); @@ -330,10 +437,12 @@ public void testFilterClausesWithRightExpression() { func = pinotQuery.getFilterExpression().getFunctionCall(); Assert.assertEquals(func.getOperator(), FilterKind.LESS_THAN_OR_EQUAL.name()); Assert.assertEquals(func.getOperands().get(0).getFunctionCall().getOperator(), "minus"); - Assert.assertEquals( - func.getOperands().get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperator(), "foo1"); - Assert.assertEquals( - func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperator(), "foo2"); + Assert + .assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperator(), + "foo1"); + Assert + .assertEquals(func.getOperands().get(0).getFunctionCall().getOperands().get(1).getFunctionCall().getOperator(), + "foo2"); Assert.assertEquals( func.getOperands().get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperands().get(0) .getFunctionCall().getOperator(), "bar1"); @@ -676,8 +785,8 @@ public void testParseExceptionHasCharacterPosition() { } catch (SqlCompilationException e) { // Expected Assert.assertTrue(e.getCause().getMessage().contains("at line 1, column 31."), - "Compilation exception should contain line and character for error message. Error message is " - + e.getMessage()); + "Compilation exception should contain line and character for error message. Error message is " + e + .getMessage()); return; } @@ -701,8 +810,8 @@ public void testQueryOptions() { Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 0); Assert.assertNull(pinotQuery.getQueryOptions()); - pinotQuery = CalciteSqlParser.compileToPinotQuery( - "select * from vegetables where name <> 'Brussels sprouts' OPTION (delicious=yes)"); + pinotQuery = CalciteSqlParser + .compileToPinotQuery("select * from vegetables where name <> 'Brussels sprouts' OPTION (delicious=yes)"); Assert.assertEquals(pinotQuery.getQueryOptionsSize(), 1); Assert.assertTrue(pinotQuery.getQueryOptions().containsKey("delicious")); Assert.assertEquals(pinotQuery.getQueryOptions().get("delicious"), "yes"); @@ -777,8 +886,8 @@ private void testRemoveComments(String sqlWithComments, String expectedSqlWithou @Test public void testIdentifierQuoteCharacter() { - PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery( - "select avg(attributes.age) as avg_age from person group by attributes.address_city"); + PinotQuery pinotQuery = CalciteSqlParser + .compileToPinotQuery("select avg(attributes.age) as avg_age from person group by attributes.address_city"); Assert.assertEquals( pinotQuery.getSelectList().get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperands().get(0) .getIdentifier().getName(), "attributes.age"); @@ -805,14 +914,15 @@ public void testStringLiteral() { Assert.assertEquals(groupbyList.get(1).getIdentifier().getName(), "bar"); // For UDF, string literal won't be treated as column but as LITERAL - pinotQuery = CalciteSqlParser.compileToPinotQuery( - "SELECT SUM(ADD(foo, 'bar')) FROM myTable GROUP BY sub(foo, bar), SUB(BAR, FOO)"); + pinotQuery = CalciteSqlParser + .compileToPinotQuery("SELECT SUM(ADD(foo, 'bar')) FROM myTable GROUP BY sub(foo, bar), SUB(BAR, FOO)"); selectFunctionList = pinotQuery.getSelectList(); Assert.assertEquals(selectFunctionList.size(), 1); Assert.assertEquals(selectFunctionList.get(0).getFunctionCall().getOperator(), "sum"); Assert.assertEquals(selectFunctionList.get(0).getFunctionCall().getOperands().size(), 1); - Assert.assertEquals( - selectFunctionList.get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperator(), "add"); + Assert + .assertEquals(selectFunctionList.get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperator(), + "add"); Assert.assertEquals( selectFunctionList.get(0).getFunctionCall().getOperands().get(0).getFunctionCall().getOperands().size(), 2); Assert.assertEquals( @@ -878,8 +988,8 @@ public void testFilterUdf() { @Test public void testSelectionTransformFunction() { - PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery( - " select mapKey(mapField,k1) from baseballStats where mapKey(mapField,k1) = 'v1'"); + PinotQuery pinotQuery = CalciteSqlParser + .compileToPinotQuery(" select mapKey(mapField,k1) from baseballStats where mapKey(mapField,k1) = 'v1'"); Assert.assertEquals(pinotQuery.getSelectList().get(0).getFunctionCall().getOperator(), "mapkey"); Assert.assertEquals( pinotQuery.getSelectList().get(0).getFunctionCall().getOperands().get(0).getIdentifier().getName(), "mapField"); @@ -1525,8 +1635,9 @@ public void testReservedKeywords() { Assert.assertEquals( pinotQuery.getSelectList().get(4).getFunctionCall().getOperands().get(1).getIdentifier().getName(), "avg"); Assert.assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperator(), FilterKind.EQUALS.name()); - Assert.assertEquals( - pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(), "groups"); + Assert + .assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(), + "groups"); Assert.assertEquals( pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(1).getLiteral().getStringValue(), "foo"); @@ -1946,8 +2057,8 @@ public void testCompileTimeExpression() { expression = pinotQuery.getFilterExpression(); Assert.assertNotNull(expression.getFunctionCall()); Assert.assertEquals(expression.getFunctionCall().getOperator(), "todatetime"); - Assert.assertEquals(expression.getFunctionCall().getOperands().get(0).getIdentifier().getName(), - "millisSinceEpoch"); + Assert + .assertEquals(expression.getFunctionCall().getOperands().get(0).getIdentifier().getName(), "millisSinceEpoch"); expression = CalciteSqlParser.compileToExpression("encodeUrl('key1=value 1&key2=value@!$2&key3=value%3')"); Assert.assertNotNull(expression.getFunctionCall()); @@ -2047,8 +2158,9 @@ public void testCaseInsensitiveFilter() { pinotQuery = CalciteSqlParser.compileToPinotQuery(query); brokerRequest = converter.convert(pinotQuery); Assert.assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperator(), "IS_NOT_NULL"); - Assert.assertEquals( - pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(), "col"); + Assert + .assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(), + "col"); Assert.assertEquals(brokerRequest.getFilterQuery().getOperator(), FilterOperator.IS_NOT_NULL); Assert.assertEquals(brokerRequest.getFilterQuery().getColumn(), "col"); @@ -2056,8 +2168,9 @@ public void testCaseInsensitiveFilter() { pinotQuery = CalciteSqlParser.compileToPinotQuery(query); brokerRequest = converter.convert(pinotQuery); Assert.assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperator(), "IS_NOT_NULL"); - Assert.assertEquals( - pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(), "col"); + Assert + .assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(), + "col"); Assert.assertEquals(brokerRequest.getFilterQuery().getOperator(), FilterOperator.IS_NOT_NULL); Assert.assertEquals(brokerRequest.getFilterQuery().getColumn(), "col"); @@ -2065,8 +2178,9 @@ public void testCaseInsensitiveFilter() { pinotQuery = CalciteSqlParser.compileToPinotQuery(query); brokerRequest = converter.convert(pinotQuery); Assert.assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperator(), "IS_NULL"); - Assert.assertEquals( - pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(), "col"); + Assert + .assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(), + "col"); Assert.assertEquals(brokerRequest.getFilterQuery().getOperator(), FilterOperator.IS_NULL); Assert.assertEquals(brokerRequest.getFilterQuery().getColumn(), "col"); @@ -2074,8 +2188,9 @@ public void testCaseInsensitiveFilter() { pinotQuery = CalciteSqlParser.compileToPinotQuery(query); brokerRequest = converter.convert(pinotQuery); Assert.assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperator(), "IS_NULL"); - Assert.assertEquals( - pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(), "col"); + Assert + .assertEquals(pinotQuery.getFilterExpression().getFunctionCall().getOperands().get(0).getIdentifier().getName(), + "col"); Assert.assertEquals(brokerRequest.getFilterQuery().getOperator(), FilterOperator.IS_NULL); Assert.assertEquals(brokerRequest.getFilterQuery().getColumn(), "col"); } @@ -2206,17 +2321,41 @@ public void testFlattenAndOr() { } { - String query = "SELECT * FROM foo WHERE col1 > 0 AND (col2 AND col3 > 0) AND col4"; + String query = "SELECT * FROM foo WHERE col1 > 0 AND (col2 AND col3 > 0) AND startsWith(col4, 'myStr')"; PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery(query); Function functionCall = pinotQuery.getFilterExpression().getFunctionCall(); Assert.assertEquals(functionCall.getOperator(), FilterKind.AND.name()); List operands = functionCall.getOperands(); Assert.assertEquals(operands.size(), 4); Assert.assertEquals(operands.get(0).getFunctionCall().getOperator(), FilterKind.GREATER_THAN.name()); - Assert.assertEquals(operands.get(1).getIdentifier().getName(), "col2"); + Assert.assertEquals(operands.get(1).getFunctionCall().getOperator(), FilterKind.EQUALS.name()); + List eqOperands = operands.get(1).getFunctionCall().getOperands(); + Assert.assertEquals(eqOperands.get(0).getIdentifier(), new Identifier("col2")); + Assert.assertEquals(eqOperands.get(1).getLiteral(), Literal.boolValue(true)); Assert.assertEquals(operands.get(2).getFunctionCall().getOperator(), FilterKind.GREATER_THAN.name()); - Assert.assertEquals(operands.get(3).getIdentifier().getName(), "col4"); + Assert.assertEquals(operands.get(3).getFunctionCall().getOperator(), FilterKind.EQUALS.name()); + eqOperands = operands.get(3).getFunctionCall().getOperands(); + Assert.assertEquals(eqOperands.get(0).getFunctionCall().getOperator(), "startswith"); + Assert.assertEquals(eqOperands.get(1).getLiteral(), Literal.boolValue(true)); + } + { + String query = "SELECT * FROM foo WHERE col1 > 0 AND (col2 AND col3 > 0) AND col4 = true"; + PinotQuery pinotQuery = CalciteSqlParser.compileToPinotQuery(query); + Function functionCall = pinotQuery.getFilterExpression().getFunctionCall(); + Assert.assertEquals(functionCall.getOperator(), FilterKind.AND.name()); + List operands = functionCall.getOperands(); + Assert.assertEquals(operands.size(), 4); + Assert.assertEquals(operands.get(0).getFunctionCall().getOperator(), FilterKind.GREATER_THAN.name()); + Assert.assertEquals(operands.get(1).getFunctionCall().getOperator(), FilterKind.EQUALS.name()); + List eqOperands = operands.get(1).getFunctionCall().getOperands(); + Assert.assertEquals(eqOperands.get(0).getIdentifier(), new Identifier("col2")); + Assert.assertEquals(eqOperands.get(1).getLiteral(), Literal.boolValue(true)); + Assert.assertEquals(operands.get(2).getFunctionCall().getOperator(), FilterKind.GREATER_THAN.name()); + Assert.assertEquals(operands.get(3).getFunctionCall().getOperator(), FilterKind.EQUALS.name()); + eqOperands = operands.get(3).getFunctionCall().getOperands(); + Assert.assertEquals(eqOperands.get(0).getIdentifier(), new Identifier("col4")); + Assert.assertEquals(eqOperands.get(1).getLiteral(), Literal.stringValue("true")); // NOTE: PQL does not support logical identifier } @@ -2249,9 +2388,12 @@ public void testFlattenAndOr() { List operands = functionCall.getOperands(); Assert.assertEquals(operands.size(), 4); Assert.assertEquals(operands.get(0).getFunctionCall().getOperator(), FilterKind.LESS_THAN_OR_EQUAL.name()); - Assert.assertEquals(operands.get(1).getIdentifier().getName(), "col2"); + Assert.assertEquals(operands.get(1).getFunctionCall().getOperator(), FilterKind.EQUALS.name()); Assert.assertEquals(operands.get(2).getFunctionCall().getOperator(), FilterKind.LESS_THAN_OR_EQUAL.name()); - Assert.assertEquals(operands.get(3).getIdentifier().getName(), "col4"); + Assert.assertEquals(operands.get(3).getFunctionCall().getOperator(), FilterKind.EQUALS.name()); + List eqOperands = operands.get(3).getFunctionCall().getOperands(); + Assert.assertEquals(eqOperands.get(0).getIdentifier(), new Identifier("col4")); + Assert.assertEquals(eqOperands.get(1).getLiteral(), Literal.boolValue(true)); // NOTE: PQL does not support logical identifier } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/request/context/QueryContext.java b/pinot-core/src/main/java/org/apache/pinot/core/query/request/context/QueryContext.java index 3121fe21862..1076a8504f0 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/request/context/QueryContext.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/request/context/QueryContext.java @@ -582,9 +582,6 @@ private static void getAggregations(ExpressionContext expression, Preconditions.checkState(aggregation != null && aggregation.getType() == FunctionContext.Type.AGGREGATION, "First argument of FILTER must be an aggregation function"); ExpressionContext filterExpression = arguments.get(1); - Preconditions.checkState(filterExpression.getFunction() != null - && filterExpression.getFunction().getType() == FunctionContext.Type.TRANSFORM, - "Second argument of FILTER must be a filter expression"); FilterContext filter = RequestContextUtils.getFilter(filterExpression); filteredAggregations.add(Pair.of(aggregation, filter)); } else { diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/BooleanQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/BooleanQueriesTest.java index dad1ac14558..384a8989eda 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/BooleanQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/BooleanQueriesTest.java @@ -162,6 +162,21 @@ public void testQueries() { assertEquals(row[0], false); } } + { + String query = "SELECT booleanColumn FROM testTable WHERE booleanColumn"; + BrokerResponseNative brokerResponse = getBrokerResponse(query); + ResultTable resultTable = brokerResponse.getResultTable(); + DataSchema dataSchema = resultTable.getDataSchema(); + assertEquals(dataSchema, + new DataSchema(new String[]{"booleanColumn"}, new ColumnDataType[]{ColumnDataType.BOOLEAN})); + List rows = resultTable.getRows(); + assertEquals(rows.size(), 10); + for (int i = 0; i < 10; i++) { + Object[] row = rows.get(i); + assertEquals(row.length, 1); + assertEquals(row[0], true); + } + } { String query = "SELECT * FROM testTable ORDER BY booleanColumn DESC LIMIT 20"; BrokerResponseNative brokerResponse = getBrokerResponse(query); diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/ExplainPlanQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/ExplainPlanQueriesTest.java index 9c2fe3d9406..7e5dc8e9aad 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/ExplainPlanQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/ExplainPlanQueriesTest.java @@ -55,6 +55,7 @@ public class ExplainPlanQueriesTest extends BaseQueriesTest { private final static String COL1_NO_INDEX = "noIndexCol1"; private final static String COL2_NO_INDEX = "noIndexCol2"; private final static String COL3_NO_INDEX = "noIndexCol3"; + private final static String COL4_NO_INDEX = "noIndexCol4"; private final static String COL1_INVERTED_INDEX = "invertedIndexCol1"; private final static String COL2_INVERTED_INDEX = "invertedIndexCol2"; private final static String COL3_INVERTED_INDEX = "invertedIndexCol3"; @@ -69,6 +70,7 @@ public class ExplainPlanQueriesTest extends BaseQueriesTest { .addSingleValueDimension(COL1_NO_INDEX, FieldSpec.DataType.INT) .addSingleValueDimension(COL2_NO_INDEX, FieldSpec.DataType.INT) .addSingleValueDimension(COL3_NO_INDEX, FieldSpec.DataType.INT) + .addSingleValueDimension(COL4_NO_INDEX, FieldSpec.DataType.BOOLEAN) .addSingleValueDimension(COL1_INVERTED_INDEX, FieldSpec.DataType.DOUBLE) .addSingleValueDimension(COL2_INVERTED_INDEX, FieldSpec.DataType.INT) .addSingleValueDimension(COL3_INVERTED_INDEX, FieldSpec.DataType.STRING) @@ -104,14 +106,16 @@ protected List getIndexSegments() { return _indexSegments; } - GenericRow createMockRecord(int noIndexCol1, int noIndexCol2, int noIndexCol3, double invertedIndexCol1, - int invertedIndexCol2, String intervedIndexCol3, double rangeIndexCol1, int rangeIndexCol2, int rangeIndexCol3, - double sortedIndexCol1, String jsonIndexCol1, String textIndexCol1) { + GenericRow createMockRecord(int noIndexCol1, int noIndexCol2, int noIndexCol3, + boolean noIndexCol4, double invertedIndexCol1, int invertedIndexCol2, String intervedIndexCol3, + double rangeIndexCol1, int rangeIndexCol2, int rangeIndexCol3, double sortedIndexCol1, String jsonIndexCol1, + String textIndexCol1) { GenericRow record = new GenericRow(); record.putValue(COL1_NO_INDEX, noIndexCol1); record.putValue(COL2_NO_INDEX, noIndexCol2); record.putValue(COL3_NO_INDEX, noIndexCol3); + record.putValue(COL4_NO_INDEX, noIndexCol4); record.putValue(COL1_INVERTED_INDEX, invertedIndexCol1); record.putValue(COL2_INVERTED_INDEX, invertedIndexCol2); @@ -135,11 +139,11 @@ public void setUp() FileUtils.deleteDirectory(INDEX_DIR); List records = new ArrayList<>(NUM_RECORDS); - records.add(createMockRecord(1, 2, 3, 1.1, 2, "daffy", 10.1, 20, 30, 100.1, + records.add(createMockRecord(1, 2, 3, true, 1.1, 2, "daffy", 10.1, 20, 30, 100.1, "{\"first\": \"daffy\", \"last\": " + "\"duck\"}", "daffy")); - records.add(createMockRecord(0, 1, 2, 0.1, 1, "mickey", 0.1, 10, 20, 100.2, + records.add(createMockRecord(0, 1, 2, false, 0.1, 1, "mickey", 0.1, 10, 20, 100.2, "{\"first\": \"mickey\", \"last\": " + "\"mouse\"}", "mickey")); - records.add(createMockRecord(3, 4, 5, 2.1, 3, "mickey", 20.1, 30, 40, 100.3, + records.add(createMockRecord(3, 4, 5, true, 2.1, 3, "mickey", 20.1, 30, 40, 100.3, "{\"first\": \"mickey\", \"last\": " + "\"mouse\"}", "mickey")); IndexingConfig indexingConfig = TABLE_CONFIG.getIndexingConfig(); @@ -191,14 +195,14 @@ public void testSelect() { result1.add(new Object[]{"COMBINE_SELECT", 1, 0}); result1.add(new Object[]{ "SELECT(selectList:invertedIndexCol1, invertedIndexCol2, invertedIndexCol3, jsonIndexCol1, " - + "noIndexCol1, noIndexCol2, noIndexCol3, rangeIndexCol1, rangeIndexCol2, rangeIndexCol3, " + + "noIndexCol1, noIndexCol2, noIndexCol3, noIndexCol4, rangeIndexCol1, rangeIndexCol2, rangeIndexCol3, " + "sortedIndexCol1, textIndexCol1)", 2, 1}); result1.add(new Object[]{"TRANSFORM_PASSTHROUGH(invertedIndexCol1, invertedIndexCol2, invertedIndexCol3, " - + "jsonIndexCol1, noIndexCol1, noIndexCol2, noIndexCol3, rangeIndexCol1, rangeIndexCol2, rangeIndexCol3, " - + "sortedIndexCol1, textIndexCol1)", 3, 2}); - result1.add(new Object[]{"PROJECT(sortedIndexCol1, noIndexCol3, rangeIndexCol1, rangeIndexCol2, jsonIndexCol1, " - + "invertedIndexCol1, noIndexCol2, invertedIndexCol2, noIndexCol1, invertedIndexCol3, rangeIndexCol3, " - + "textIndexCol1)", 4, 3}); + + "jsonIndexCol1, noIndexCol1, noIndexCol2, noIndexCol3, noIndexCol4, rangeIndexCol1, rangeIndexCol2, " + + "rangeIndexCol3, sortedIndexCol1, textIndexCol1)", 3, 2}); + result1.add(new Object[]{"PROJECT(noIndexCol4, sortedIndexCol1, noIndexCol3, rangeIndexCol1, rangeIndexCol2, " + + "invertedIndexCol1, noIndexCol2, invertedIndexCol2, noIndexCol1, rangeIndexCol3, textIndexCol1, " + + "jsonIndexCol1, invertedIndexCol3)", 4, 3}); result1.add(new Object[]{"DOC_ID_SET", 5, 4}); result1.add(new Object[]{"FILTER_MATCH_ENTIRE_SEGMENT(docs:3)", 6, 5}); check(query1, new ResultTable(DATA_SCHEMA, result1)); @@ -337,6 +341,47 @@ public void testSelectColumnsUsingFilter() { result3.add(new Object[]{"FILTER_FULL_SCAN(operator:RANGE,predicate:noIndexCol1 > '1')", 7, 6}); result3.add(new Object[]{"FILTER_FULL_SCAN(operator:RANGE,predicate:noIndexCol2 BETWEEN '2' AND '101')", 8, 6}); check(query3, new ResultTable(DATA_SCHEMA, result3)); + + String query4 = "EXPLAIN PLAN FOR SELECT invertedIndexCol1, noIndexCol1 FROM testTable WHERE noIndexCol1 > 1 OR " + + "contains(textIndexCol1, 'daff') OR noIndexCol2 BETWEEN 2 AND 101 LIMIT 100"; + List result4 = new ArrayList<>(); + result4.add(new Object[]{"BROKER_REDUCE(limit:100)", 0, -1}); + result4.add(new Object[]{"COMBINE_SELECT", 1, 0}); + result4.add(new Object[]{"SELECT(selectList:invertedIndexCol1, noIndexCol1)", 2, 1}); + result4.add(new Object[]{"TRANSFORM_PASSTHROUGH(invertedIndexCol1, noIndexCol1)", 3, 2}); + result4.add(new Object[]{"PROJECT(invertedIndexCol1, noIndexCol1)", 4, 3}); + result4.add(new Object[]{"DOC_ID_SET", 5, 4}); + result4.add(new Object[]{"FILTER_OR", 6, 5}); + result4.add(new Object[]{"FILTER_FULL_SCAN(operator:RANGE,predicate:noIndexCol1 > '1')", 7, 6}); + result4.add(new Object[]{"FILTER_EXPRESSION(operator:EQ,predicate:contains(textIndexCol1,'daff') = 'true')", 8, 6}); + result4.add(new Object[]{"FILTER_FULL_SCAN(operator:RANGE,predicate:noIndexCol2 BETWEEN '2' AND '101')", 9, 6}); + check(query4, new ResultTable(DATA_SCHEMA, result4)); + + String query5 = "EXPLAIN PLAN FOR SELECT invertedIndexCol1, noIndexCol1 FROM testTable WHERE noIndexCol4 LIMIT 100"; + List result5 = new ArrayList<>(); + result5.add(new Object[]{"BROKER_REDUCE(limit:100)", 0, -1}); + result5.add(new Object[]{"COMBINE_SELECT", 1, 0}); + result5.add(new Object[]{"SELECT(selectList:invertedIndexCol1, noIndexCol1)", 2, 1}); + result5.add(new Object[]{"TRANSFORM_PASSTHROUGH(invertedIndexCol1, noIndexCol1)", 3, 2}); + result5.add(new Object[]{"PROJECT(invertedIndexCol1, noIndexCol1)", 4, 3}); + result5.add(new Object[]{"DOC_ID_SET", 5, 4}); + result5.add(new Object[]{"FILTER_FULL_SCAN(operator:EQ,predicate:noIndexCol4 = 'true')", 6, 5}); + check(query5, new ResultTable(DATA_SCHEMA, result5)); + + String query6 = "EXPLAIN PLAN FOR SELECT invertedIndexCol1, noIndexCol1 FROM testTable WHERE startsWith " + + "(textIndexCol1, 'daff') AND noIndexCol4"; + List result6 = new ArrayList<>(); + result6.add(new Object[]{"BROKER_REDUCE(limit:10)", 0, -1}); + result6.add(new Object[]{"COMBINE_SELECT", 1, 0}); + result6.add(new Object[]{"SELECT(selectList:invertedIndexCol1, noIndexCol1)", 2, 1}); + result6.add(new Object[]{"TRANSFORM_PASSTHROUGH(invertedIndexCol1, noIndexCol1)", 3, 2}); + result6.add(new Object[]{"PROJECT(invertedIndexCol1, noIndexCol1)", 4, 3}); + result6.add(new Object[]{"DOC_ID_SET", 5, 4}); + result6.add(new Object[]{"FILTER_AND", 6, 5}); + result6.add(new Object[]{"FILTER_FULL_SCAN(operator:EQ,predicate:noIndexCol4 = 'true')", 7, 6}); + result6.add(new Object[]{"FILTER_EXPRESSION(operator:EQ,predicate:startswith(textIndexCol1,'daff') = 'true')", 8, + 6}); + check(query6, new ResultTable(DATA_SCHEMA, result6)); } /** Test case for SQL statements with filter that involves inverted or sorted index access. */ diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/FilteredAggregationsTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/FilteredAggregationsTest.java index 5b3a1bc153b..2fc9ad1fa6d 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/FilteredAggregationsTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/FilteredAggregationsTest.java @@ -26,6 +26,8 @@ import java.util.List; import java.util.Set; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang.math.RandomUtils; import org.apache.pinot.common.response.broker.ResultTable; import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoader; import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl; @@ -57,6 +59,8 @@ public class FilteredAggregationsTest extends BaseQueriesTest { private static final String INT_COL_NAME = "INT_COL"; private static final String NO_INDEX_INT_COL_NAME = "NO_INDEX_COL"; private static final String STATIC_INT_COL_NAME = "STATIC_INT_COL"; + private static final String BOOLEAN_COL_NAME = "BOOLEAN_COL"; + private static final String STRING_COL_NAME = "STRING_COL"; private static final Integer NUM_ROWS = 30000; private IndexSegment _indexSegment; @@ -111,6 +115,8 @@ private List createTestData() { row.putValue(INT_COL_NAME, i); row.putValue(NO_INDEX_INT_COL_NAME, i); row.putValue(STATIC_INT_COL_NAME, 10); + row.putValue(BOOLEAN_COL_NAME, RandomUtils.nextBoolean()); + row.putValue(STRING_COL_NAME, RandomStringUtils.randomAlphabetic(4)); rows.add(row); } return rows; @@ -126,6 +132,8 @@ private void buildSegment(String segmentName) Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME) .addSingleValueDimension(NO_INDEX_INT_COL_NAME, FieldSpec.DataType.INT) .addSingleValueDimension(STATIC_INT_COL_NAME, FieldSpec.DataType.INT) + .addSingleValueDimension(BOOLEAN_COL_NAME, FieldSpec.DataType.BOOLEAN) + .addSingleValueDimension(STRING_COL_NAME, FieldSpec.DataType.STRING) .addMetric(INT_COL_NAME, FieldSpec.DataType.INT).build(); SegmentGeneratorConfig config = new SegmentGeneratorConfig(tableConfig, schema); config.setOutDir(INDEX_DIR.getPath()); @@ -185,6 +193,20 @@ public void testSimpleQueries() { + "FROM MyTable"; nonFilterQuery = "SELECT MIN(INT_COL), MAX(INT_COL) FROM MyTable WHERE INT_COL > 29990"; testQuery(filterQuery, nonFilterQuery); + + filterQuery = "SELECT SUM(INT_COL) FILTER(WHERE BOOLEAN_COL) FROM MyTable"; + nonFilterQuery = "SELECT SUM(INT_COL) FROM MyTable WHERE BOOLEAN_COL=true"; + testQuery(filterQuery, nonFilterQuery); + + filterQuery = "SELECT SUM(INT_COL) FILTER(WHERE BOOLEAN_COL AND STARTSWITH(STRING_COL, 'abc')) FROM MyTable"; + nonFilterQuery = "SELECT SUM(INT_COL) FROM MyTable WHERE BOOLEAN_COL=true AND STARTSWITH(STRING_COL, 'abc')"; + testQuery(filterQuery, nonFilterQuery); + + filterQuery = "SELECT SUM(INT_COL) FILTER(WHERE BOOLEAN_COL AND STARTSWITH(REVERSE(STRING_COL), 'abc')) FROM " + + "MyTable"; + nonFilterQuery = "SELECT SUM(INT_COL) FROM MyTable WHERE BOOLEAN_COL=true AND STARTSWITH(REVERSE(STRING_COL), " + + "'abc')"; + testQuery(filterQuery, nonFilterQuery); } @Test