From 2c3925acdc1be1143cd2aedc3c758c26e6a91b8a Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 30 Apr 2024 22:03:33 -0700 Subject: [PATCH] Fallback vectorization for FunctionExpr and BaseMacroFunctionExpr. This patch adds FallbackVectorProcessor, a processor that adapts non-vectorizable operations into vectorizable ones. It is used in FunctionExpr and BaseMacroFunctionExpr. In addition: - Identifiers are updated to offer getObjectVector for ARRAY and COMPLEX in addition to STRING. ExprEvalObjectVector is updated to offer ARRAY and COMPLEX as well. - In SQL tests, cannotVectorize now fails tests if an exception is not thrown. This makes it easier to identify tests that can now vectorize. - Fix a null-matcher bug in StringObjectVectorValueMatcher. --- .../druid/math/expr/ExprMacroTable.java | 14 + .../druid/math/expr/FunctionalExpr.java | 10 +- .../vector/CastToStringVectorProcessor.java | 2 +- .../expr/vector/ExprEvalObjectVector.java | 39 +- .../expr/vector/FallbackVectorProcessor.java | 396 ++++++++++++++++++ ...ObjectOutMultiObjectInVectorProcessor.java | 2 +- ...ctOutObjectsInFunctionVectorProcessor.java | 2 +- .../math/expr/vector/VectorProcessors.java | 12 +- .../StringObjectVectorValueMatcher.java | 2 +- .../virtual/ExpressionVectorSelectors.java | 2 +- .../vector/FallbackVectorProcessorTest.java | 294 +++++++++++++ .../ExpressionVectorObjectSelectorTest.java | 3 +- .../sql/calcite/CalciteArraysQueryTest.java | 81 +--- .../sql/calcite/CalciteJoinQueryTest.java | 5 - .../CalciteLookupFunctionQueryTest.java | 28 -- .../CalciteMultiValueStringQueryTest.java | 14 - .../calcite/CalciteNestedDataQueryTest.java | 10 - .../calcite/CalciteParameterQueryTest.java | 5 - .../druid/sql/calcite/CalciteQueryTest.java | 137 +----- .../sql/calcite/CalciteSelectQueryTest.java | 14 - .../sql/calcite/CalciteSubqueryTest.java | 30 +- .../calcite/CalciteTimeBoundaryQueryTest.java | 6 - .../druid/sql/calcite/QueryTestRunner.java | 13 +- 23 files changed, 793 insertions(+), 328 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/FallbackVectorProcessor.java create mode 100644 processing/src/test/java/org/apache/druid/math/expr/vector/FallbackVectorProcessorTest.java diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java b/processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java index 4910baafefba..a9f5b941c78f 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java +++ b/processing/src/main/java/org/apache/druid/math/expr/ExprMacroTable.java @@ -25,6 +25,8 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.FallbackVectorProcessor; import javax.annotation.Nullable; import java.util.Collections; @@ -158,6 +160,18 @@ public BindingAnalysis analyzeInputs() return analyzeInputsSupplier.get(); } + @Override + public boolean canVectorize(InputBindingInspector inspector) + { + return getOutputType(inspector) != null && inspector.canVectorize(args); + } + + @Override + public ExprVectorProcessor asVectorProcessor(VectorInputBindingInspector inspector) + { + return FallbackVectorProcessor.create(macro, args, inspector); + } + /** * Implemented by subclasses to provide the value for {@link #analyzeInputs()}, which uses a memoized supplier. */ diff --git a/processing/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java b/processing/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java index 6fe762a63374..64dd9f78c208 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java +++ b/processing/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java @@ -24,6 +24,7 @@ import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.FallbackVectorProcessor; import org.apache.druid.segment.column.Types; import javax.annotation.Nullable; @@ -223,13 +224,18 @@ public ExprEval eval(ObjectBinding bindings) @Override public boolean canVectorize(InputBindingInspector inspector) { - return function.canVectorize(inspector, args); + return function.canVectorize(inspector, args) + || (getOutputType(inspector) != null && inspector.canVectorize(args)); } @Override public ExprVectorProcessor asVectorProcessor(VectorInputBindingInspector inspector) { - return function.asVectorProcessor(inspector, args); + if (function.canVectorize(inspector, args)) { + return function.asVectorProcessor(inspector, args); + } else { + return FallbackVectorProcessor.create(function, args, inspector); + } } @Override diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java index b02afd4f88f6..f0e2dc7ee42a 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java @@ -39,7 +39,7 @@ public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) for (int i = 0; i < objects.length; i++) { output[i] = Evals.asString(objects[i]); } - return new ExprEvalObjectVector(output); + return new ExprEvalObjectVector(output, ExpressionType.STRING); } @Override diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalObjectVector.java b/processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalObjectVector.java index eaeb0f04f1ef..cc22445358fe 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalObjectVector.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalObjectVector.java @@ -20,8 +20,10 @@ package org.apache.druid.math.expr.vector; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.error.DruidException; import org.apache.druid.math.expr.Evals; import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExprType; import org.apache.druid.math.expr.ExpressionType; import javax.annotation.Nullable; @@ -36,9 +38,17 @@ public final class ExprEvalObjectVector extends ExprEvalVector @Nullable private boolean[] numericNulls; - public ExprEvalObjectVector(Object[] values) + private final ExpressionType type; + + public ExprEvalObjectVector(Object[] values, ExpressionType type) { super(values, null); + this.type = type; + + if (type.isNumeric()) { + // Cannot use ExprEvalObjectSelector on types that are innately numbers. + throw DruidException.defensive("Expression of type[%s] is numeric", type); + } } private void computeNumbers() @@ -47,16 +57,25 @@ private void computeNumbers() longs = new long[values.length]; doubles = new double[values.length]; numericNulls = new boolean[values.length]; + boolean isString = type.is(ExprType.STRING); for (int i = 0; i < values.length; i++) { - Number n = ExprEval.computeNumber(Evals.asString(values[i])); - if (n != null) { - longs[i] = n.longValue(); - doubles[i] = n.doubleValue(); - numericNulls[i] = false; + if (isString) { + Number n = ExprEval.computeNumber(Evals.asString(values[i])); + if (n != null) { + longs[i] = n.longValue(); + doubles[i] = n.doubleValue(); + numericNulls[i] = false; + } else { + longs[i] = 0L; + doubles[i] = 0.0; + numericNulls[i] = NullHandling.sqlCompatible(); + } } else { - longs[i] = 0L; - doubles[i] = 0.0; - numericNulls[i] = NullHandling.sqlCompatible(); + // ARRAY, COMPLEX + final ExprEval valueEval = ExprEval.bestEffortOf(values[i]); + longs[i] = valueEval.asLong(); + doubles[i] = valueEval.asDouble(); + numericNulls[i] = valueEval.isNumericNull(); } } } @@ -73,7 +92,7 @@ public boolean[] getNullVector() @Override public ExpressionType getType() { - return ExpressionType.STRING; + return type; } @Override diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/FallbackVectorProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/FallbackVectorProcessor.java new file mode 100644 index 000000000000..27e4d154a5d3 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/FallbackVectorProcessor.java @@ -0,0 +1,396 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.error.DruidException; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.math.expr.ExprType; +import org.apache.druid.math.expr.ExpressionType; +import org.apache.druid.math.expr.Function; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +/** + * Implementation of {@link ExprVectorProcessor} that adapts non-vectorized {@link Expr#eval(Expr.ObjectBinding)}. + * This allows non-vectorized expressions to participate in vectorized queries. + */ +public abstract class FallbackVectorProcessor implements ExprVectorProcessor +{ + final Supplier> fn; + final List adaptedArgs; + + private final ExpressionType outputType; + + private FallbackVectorProcessor( + final Supplier> fn, + final List adaptedArgs, + final ExpressionType outputType + ) + { + this.fn = fn; + this.adaptedArgs = adaptedArgs; + this.outputType = outputType; + } + + /** + * Create a processor for a non-vectorizable {@link Function}. + */ + public static FallbackVectorProcessor create( + final Function function, + final List args, + final Expr.VectorInputBindingInspector inspector + ) + { + final List adaptedArgs = makeAdaptedArgs(args, inspector); + return makeFallbackProcessor( + () -> function.apply(adaptedArgs, UnusedBinding.INSTANCE), + adaptedArgs, + function.getOutputType(inspector, args), + inspector + ); + } + + /** + * Create a processor for a non-vectorizable {@link ExprMacroTable.ExprMacro}. + */ + public static FallbackVectorProcessor create( + final ExprMacroTable.ExprMacro macro, + final List args, + final Expr.VectorInputBindingInspector inspector + ) + { + final List adaptedArgs = makeAdaptedArgs(args, inspector); + final Expr adaptedExpr = macro.apply(adaptedArgs); + return makeFallbackProcessor( + () -> adaptedExpr.eval(UnusedBinding.INSTANCE), + adaptedArgs, + adaptedExpr.getOutputType(inspector), + inspector + ); + } + + /** + * Helper for the two {@link #create} methods. Makes {@link AdaptedExpr} that can replace the original args to + * the {@link Expr} that requires fallback. + * + * @param args args to the original expr + * @param inspector binding inspector + * + * @return list of {@link AdaptedExpr} + */ + private static List makeAdaptedArgs( + final List args, + final Expr.VectorInputBindingInspector inspector + ) + { + final List adaptedArgs = new ArrayList<>(args.size()); + + for (final Expr arg : args) { + adaptedArgs.add(new AdaptedExpr(arg.asVectorProcessor(inspector), arg)); + } + + return adaptedArgs; + } + + /** + * Helper for the two {@link #create} methods. + * + * @param fn eval function that uses the "adaptedArgs" as inputs + * @param adaptedArgs adapted args from {@link #makeAdaptedArgs(List, Expr.VectorInputBindingInspector)} + * @param outputType output type of the eval from "fn" + * @param inspector binding inspector + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + private static FallbackVectorProcessor makeFallbackProcessor( + final Supplier> fn, + final List adaptedArgs, + final ExpressionType outputType, + final Expr.VectorInputBindingInspector inspector + ) + { + if (outputType == null) { + throw DruidException.defensive("Plan has null outputType"); + } else if (outputType.equals(ExpressionType.LONG)) { + return (FallbackVectorProcessor) new OfLong(fn, (List) adaptedArgs, outputType, inspector); + } else if (outputType.equals(ExpressionType.DOUBLE)) { + return (FallbackVectorProcessor) new OfDouble(fn, (List) adaptedArgs, outputType, inspector); + } else { + return (FallbackVectorProcessor) new OfObject(fn, (List) adaptedArgs, outputType, inspector); + } + } + + @Override + public ExpressionType getOutputType() + { + return outputType; + } + + /** + * Specialization for non-numeric types. + */ + private static class OfObject extends FallbackVectorProcessor + { + private final Object[] outValues; + + public OfObject( + final Supplier> fn, + final List args, + final ExpressionType outputType, + final Expr.VectorInputBindingInspector inspector + ) + { + super(fn, args, outputType); + this.outValues = new Object[inspector.getMaxVectorSize()]; + } + + @Override + public ExprEvalVector evalVector(Expr.VectorInputBinding vectorBindings) + { + for (final AdaptedExpr adaptedArg : adaptedArgs) { + adaptedArg.populate(vectorBindings); + } + + final int sz = vectorBindings.getCurrentVectorSize(); + for (int i = 0; i < sz; i++) { + for (final AdaptedExpr adaptedArg : adaptedArgs) { + adaptedArg.setRowNumber(i); + } + + outValues[i] = fn.get().value(); + } + + return new ExprEvalObjectVector(outValues, getOutputType()); + } + } + + /** + * Specialization for {@link ExpressionType#LONG}. + */ + private static class OfLong extends FallbackVectorProcessor + { + private final long[] outValues; + private final boolean[] outNulls; + + public OfLong( + final Supplier> fn, + final List args, + final ExpressionType outputType, + final Expr.VectorInputBindingInspector inspector + ) + { + super(fn, args, outputType); + this.outValues = new long[inspector.getMaxVectorSize()]; + this.outNulls = new boolean[inspector.getMaxVectorSize()]; + } + + @Override + public ExprEvalVector evalVector(Expr.VectorInputBinding vectorBindings) + { + for (final AdaptedExpr adaptedArg : adaptedArgs) { + adaptedArg.populate(vectorBindings); + } + + final int sz = vectorBindings.getCurrentVectorSize(); + for (int i = 0; i < sz; i++) { + for (final AdaptedExpr adaptedArg : adaptedArgs) { + adaptedArg.setRowNumber(i); + } + + final ExprEval eval = fn.get(); + outValues[i] = eval.asLong(); + outNulls[i] = eval.isNumericNull(); + } + + return new ExprEvalLongVector(outValues, outNulls); + } + } + + /** + * Specialization for {@link ExpressionType#DOUBLE}. + */ + private static class OfDouble extends FallbackVectorProcessor + { + private final double[] outValues; + private final boolean[] outNulls; + + public OfDouble( + final Supplier> fn, + final List args, + final ExpressionType outputType, + final Expr.VectorInputBindingInspector inspector + ) + { + super(fn, args, outputType); + this.outValues = new double[inspector.getMaxVectorSize()]; + this.outNulls = new boolean[inspector.getMaxVectorSize()]; + } + + @Override + public ExprEvalVector evalVector(Expr.VectorInputBinding vectorBindings) + { + for (final AdaptedExpr adaptedArg : adaptedArgs) { + adaptedArg.populate(vectorBindings); + } + + final int sz = vectorBindings.getCurrentVectorSize(); + for (int i = 0; i < sz; i++) { + for (final AdaptedExpr adaptedArg : adaptedArgs) { + adaptedArg.setRowNumber(i); + } + + final ExprEval eval = fn.get(); + outValues[i] = eval.asDouble(); + outNulls[i] = eval.isNumericNull(); + } + + return new ExprEvalDoubleVector(outValues, outNulls); + } + } + + /** + * Wrapper around {@link Expr} that pulls results from a {@link ExprVectorProcessor} rather than calling + * {@link Expr#eval(ObjectBinding)}. When using {@link FallbackVectorProcessor}, adapters of this class replace + * the arguments of the original {@link Expr}. + */ + private static class AdaptedExpr implements Expr + { + private final ExprVectorProcessor processor; + private final Expr originalExpr; + private final ExpressionType type; + + private ExprEvalVector results; + private int rowNum; + + public AdaptedExpr(final ExprVectorProcessor processor, final Expr originalExpr) + { + this.processor = processor; + this.originalExpr = originalExpr; + this.type = processor.getOutputType(); + } + + /** + * Populate the {@link #results} vector. Called once per vector. + */ + public void populate(final Expr.VectorInputBinding vectorBindings) + { + this.results = processor.evalVector(vectorBindings); + } + + /** + * Set {@link #rowNum}, which controls which row of {@link #results} is returned by {@link #eval(ObjectBinding)}. + */ + public void setRowNumber(final int rowNum) + { + this.rowNum = rowNum; + } + + @Override + public ExprEval eval(ObjectBinding bindings) + { + if (results == null) { + // "results" can be null if eval is called during ExprMacro#apply. + return originalExpr.eval(bindings); + } + + // In all other cases, ignore the provided bindings and use the computed "results" instead. + if (type.is(ExprType.LONG)) { + final boolean isNull = results.getNullVector() != null && results.getNullVector()[rowNum]; + return ExprEval.ofLong(isNull ? null : results.getLongVector()[rowNum]); + } else if (type.is(ExprType.DOUBLE)) { + final boolean isNull = results.getNullVector() != null && results.getNullVector()[rowNum]; + return ExprEval.ofDouble(isNull ? null : results.getDoubleVector()[rowNum]); + } else { + return ExprEval.ofType(type, results.getObjectVector()[rowNum]); + } + } + + @Override + public String stringify() + { + throw DruidException.defensive( + "Unexpected call to stringify in fallback processor for expr[%s]", + originalExpr.stringify() + ); + } + + @Override + public Expr visit(Shuttle shuttle) + { + throw DruidException.defensive( + "Unexpected call to visit in fallback processor for expr[%s]", + originalExpr.stringify() + ); + } + + @Override + public BindingAnalysis analyzeInputs() + { + return originalExpr.analyzeInputs(); + } + + @Override + public boolean isLiteral() + { + return originalExpr.isLiteral(); + } + + @Override + public boolean isNullLiteral() + { + return originalExpr.isNullLiteral(); + } + + @Nullable + @Override + public Object getLiteralValue() + { + return originalExpr.getLiteralValue(); + } + } + + /** + * Implementation of {@link Expr.ObjectBinding} where we do not actually expect any methods to be called. This is + * because bindings should only be used by identifiers, and this fallback processor is never used to + * implement identifiers. + */ + private static class UnusedBinding implements Expr.ObjectBinding + { + public static final UnusedBinding INSTANCE = new UnusedBinding(); + + @Nullable + @Override + public Object get(String name) + { + throw DruidException.defensive("Unexpected binding.get call for field[%s]", name); + } + + @Nullable + @Override + public ExpressionType getType(String name) + { + throw DruidException.defensive("Unexpected binding.getType call for field[%s]", name); + } + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/ObjectOutMultiObjectInVectorProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/ObjectOutMultiObjectInVectorProcessor.java index 0d668dc2900a..79e99b277c0c 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/ObjectOutMultiObjectInVectorProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/ObjectOutMultiObjectInVectorProcessor.java @@ -61,7 +61,7 @@ public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) for (int i = 0; i < currentSize; i++) { processIndex(in, i); } - return new ExprEvalObjectVector(outValues); + return new ExprEvalObjectVector(outValues, expressionType); } abstract void processIndex(Object[][] in, int i); diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/ObjectOutObjectsInFunctionVectorProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/ObjectOutObjectsInFunctionVectorProcessor.java index d64ffa384236..fc98706fe674 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/ObjectOutObjectsInFunctionVectorProcessor.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/ObjectOutObjectsInFunctionVectorProcessor.java @@ -61,7 +61,7 @@ void processNull(int i) @Override ExprEvalVector asEval() { - return new ExprEvalObjectVector(outValues); + return new ExprEvalObjectVector(outValues, expressionType); } @Override diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java b/processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java index 2e2c5ffee3ca..f81326438a16 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java @@ -89,7 +89,7 @@ public static ExprVectorProcessor constant(@Nullable String constant, int { final Object[] strings = new Object[maxVectorSize]; Arrays.fill(strings, constant); - final ExprEvalObjectVector eval = new ExprEvalObjectVector(strings); + final ExprEvalObjectVector eval = new ExprEvalObjectVector(strings, ExpressionType.STRING); return new ExprVectorProcessor() { @Override @@ -200,7 +200,7 @@ public static ExprVectorProcessor identifier(Expr.VectorInputBindingInspector @Override public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - return new ExprEvalObjectVector(bindings.getObjectVector(binding)); + return new ExprEvalObjectVector(bindings.getObjectVector(binding), ExpressionType.STRING); } }; } @@ -223,17 +223,15 @@ public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) return new ExprEvalDoubleVector(bindings.getDoubleVector(binding), bindings.getNullVector(binding)); } }; - case STRING: + default: return new IdentifierVectorProcessor(inputType) { @Override public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - return new ExprEvalObjectVector(bindings.getObjectVector(binding)); + return new ExprEvalObjectVector(bindings.getObjectVector(binding), ExpressionType.STRING); } }; - default: - throw Exprs.cannotVectorize("[" + binding + "]"); } } @@ -619,7 +617,7 @@ public void processIndex( @Override public ExprEvalVector asEval() { - return new ExprEvalObjectVector(output); + return new ExprEvalObjectVector(output, getOutputType()); } } ); diff --git a/processing/src/main/java/org/apache/druid/query/filter/vector/StringObjectVectorValueMatcher.java b/processing/src/main/java/org/apache/druid/query/filter/vector/StringObjectVectorValueMatcher.java index fa54a36a2f8e..1a882025c3e2 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/vector/StringObjectVectorValueMatcher.java +++ b/processing/src/main/java/org/apache/druid/query/filter/vector/StringObjectVectorValueMatcher.java @@ -58,7 +58,7 @@ public ReadableVectorMatch match(final ReadableVectorMatch mask, boolean include for (int i = 0; i < mask.getSelectionSize(); i++) { final int rowNum = mask.getSelection()[i]; - if ((value == null && includeUnknown) || Objects.equals(value, vector[rowNum])) { + if ((vector[rowNum] == null && includeUnknown) || Objects.equals(value, vector[rowNum])) { selection[numRows++] = rowNum; } } diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java index 66ddc2c5f8a3..654a734e3750 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java @@ -171,7 +171,7 @@ private static Expr.VectorInputBinding createVectorBindings( default: binding.addObjectSelector( columnName, - ExpressionType.STRING, + ExpressionType.fromColumnType(columnCapabilities.toColumnType()), vectorColumnSelectorFactory.makeObjectSelector(columnName) ); } diff --git a/processing/src/test/java/org/apache/druid/math/expr/vector/FallbackVectorProcessorTest.java b/processing/src/test/java/org/apache/druid/math/expr/vector/FallbackVectorProcessorTest.java new file mode 100644 index 000000000000..f921984ff9c2 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/math/expr/vector/FallbackVectorProcessorTest.java @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.math.expr.ExpressionType; +import org.apache.druid.math.expr.Function; +import org.apache.druid.math.expr.Parser; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.testing.InitializedNullHandlingTest; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public class FallbackVectorProcessorTest extends InitializedNullHandlingTest +{ + private static final int VECTOR_ID = 0; + private static final int VECTOR_SIZE = 3; + private static final RowSignature SIGNATURE = + RowSignature.builder() + .add("long", ColumnType.LONG) + .add("double", ColumnType.DOUBLE) + .add("doubleNoNulls", ColumnType.DOUBLE) + .add("str", ColumnType.STRING) + .add("arr", ColumnType.LONG_ARRAY) + .build(); + + private static final Map> DATA = + ImmutableMap.>builder() + .put("long", Arrays.asList(101L, null, 103L)) + .put("double", Arrays.asList(1.1, null, 1.3)) + .put("doubleNoNulls", Arrays.asList(1.1, 1.2, 1.3)) + .put("str", Arrays.asList("foo", null, "baz")) + .put( + "arr", + Arrays.asList( + new Object[]{201L, null, 203L}, + null, + new Object[]{301L, null, 303L} + ) + ) + .build(); + + private Expr.VectorInputBinding binding; + + @Before + public void setUp() + { + binding = new Expr.VectorInputBinding() + { + @Override + public Object[] getObjectVector(String name) + { + if (getType(name).isNumeric()) { + throw new ISE("Incorrect call for field[%s] of type[%s]", name, getType(name)); + } + + return DATA.get(name).toArray(); + } + + @Override + public long[] getLongVector(String name) + { + if (!getType(name).equals(ExpressionType.LONG)) { + throw new ISE("Incorrect call for field[%s] of type[%s]", name, getType(name)); + } + + final List longs = DATA.get(name); + final long[] longArray = new long[VECTOR_SIZE]; + + for (int i = 0; i < longs.size(); i++) { + Object o = longs.get(i); + longArray[i] = o == null ? 0 : (long) o; + } + + return longArray; + } + + @Override + public double[] getDoubleVector(String name) + { + if (!getType(name).equals(ExpressionType.DOUBLE)) { + throw new ISE("Incorrect call for field[%s] of type[%s]", name, getType(name)); + } + + final List doubles = DATA.get(name); + final double[] doubleArray = new double[VECTOR_SIZE]; + + for (int i = 0; i < doubles.size(); i++) { + Object o = doubles.get(i); + doubleArray[i] = o == null ? 0 : (double) o; + } + + return doubleArray; + } + + @Nullable + @Override + public boolean[] getNullVector(String name) + { + final List objects = DATA.get(name); + final boolean[] nullArray = new boolean[VECTOR_SIZE]; + + boolean anyNulls = false; + for (int i = 0; i < objects.size(); i++) { + Object o = objects.get(i); + nullArray[i] = o == null; + anyNulls = anyNulls || o == null; + } + + return anyNulls ? nullArray : null; + } + + @Override + public int getCurrentVectorSize() + { + return VECTOR_SIZE; + } + + @Override + public int getCurrentVectorId() + { + return VECTOR_ID; + } + + @Override + public int getMaxVectorSize() + { + return VECTOR_SIZE; + } + + @Nullable + @Override + public ExpressionType getType(String name) + { + return SIGNATURE.getColumnType(name).map(ExpressionType::fromColumnType).orElse(null); + } + }; + } + + @Test + public void test_case_long_double() + { + final FallbackVectorProcessor processor = FallbackVectorProcessor.create( + new Function.CaseSimpleFunc(), + ImmutableList.of( + Parser.parse("long + double", ExprMacroTable.nil()), + Parser.parse("102.1", ExprMacroTable.nil()), + Parser.parse("long", ExprMacroTable.nil()), + Parser.parse("double", ExprMacroTable.nil()) + ), + binding + ); + + final ExprEvalVector eval = processor.evalVector(binding); + + Assert.assertEquals(ExpressionType.DOUBLE, eval.getType()); + Assert.assertArrayEquals( + new Object[]{101.0, null, 1.3}, + eval.getObjectVector() + ); + Assert.assertArrayEquals( + new double[]{101.0, 0L, 1.3}, + eval.getDoubleVector(), + 0 + ); + Assert.assertArrayEquals( + new boolean[]{false, true, false}, + eval.getNullVector() + ); + } + + @Test + public void test_upper_string() + { + final FallbackVectorProcessor processor = FallbackVectorProcessor.create( + new Function.UpperFunc(), + ImmutableList.of( + Parser.parse("str", ExprMacroTable.nil()) + ), + binding + ); + + final ExprEvalVector eval = processor.evalVector(binding); + + Assert.assertEquals(ExpressionType.STRING, eval.getType()); + Assert.assertArrayEquals( + new Object[]{"FOO", null, "BAZ"}, + eval.getObjectVector() + ); + } + + @Test + public void test_concat_string_doubleNoNulls() + { + final FallbackVectorProcessor processor = FallbackVectorProcessor.create( + new Function.ConcatFunc(), + ImmutableList.of( + Parser.parse("str", ExprMacroTable.nil()), + Parser.parse("doubleNoNulls + 2", ExprMacroTable.nil()) + ), + binding + ); + + final ExprEvalVector eval = processor.evalVector(binding); + + Assert.assertEquals(ExpressionType.STRING, eval.getType()); + Assert.assertArrayEquals( + new Object[]{"foo3.1", null, "baz3.3"}, + eval.getObjectVector() + ); + } + + @Test + public void test_array_length() + { + final FallbackVectorProcessor processor = FallbackVectorProcessor.create( + new Function.ArrayLengthFunction(), + ImmutableList.of( + Parser.parse("arr", ExprMacroTable.nil()) + ), + binding + ); + + final ExprEvalVector eval = processor.evalVector(binding); + + Assert.assertEquals(ExpressionType.LONG, eval.getType()); + Assert.assertArrayEquals( + new Object[]{3L, null, 3L}, + eval.getObjectVector() + ); + Assert.assertArrayEquals( + new long[]{3L, 0L, 3L}, + eval.getLongVector() + ); + Assert.assertArrayEquals( + new boolean[]{false, true, false}, + eval.getNullVector() + ); + } + + @Test + public void test_array_concat() + { + final FallbackVectorProcessor processor = FallbackVectorProcessor.create( + new Function.ArrayConcatFunction(), + ImmutableList.of( + Parser.parse("arr", ExprMacroTable.nil()), + Parser.parse("long", ExprMacroTable.nil()) + ), + binding + ); + + final ExprEvalVector eval = processor.evalVector(binding); + + Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.getType()); + Assert.assertArrayEquals( + new Object[]{201L, null, 203L, 101L}, + (Object[]) eval.getObjectVector()[0] + ); + Assert.assertNull(eval.getObjectVector()[1]); + Assert.assertArrayEquals( + new Object[]{301L, null, 303L, 103L}, + (Object[]) eval.getObjectVector()[2] + ); + } +} diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelectorTest.java b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelectorTest.java index 8773bbd6cd2d..cd7e54bfdb85 100644 --- a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelectorTest.java +++ b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelectorTest.java @@ -20,6 +20,7 @@ package org.apache.druid.segment.virtual; import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExpressionType; import org.apache.druid.math.expr.vector.ExprEvalObjectVector; import org.apache.druid.math.expr.vector.ExprEvalVector; import org.apache.druid.math.expr.vector.ExprVectorProcessor; @@ -57,7 +58,7 @@ public void tearDown() public void testSelectObject() { final String[] vector = new String[]{"1", "2", null, "3"}; - ExprEvalVector vectorEval = new ExprEvalObjectVector(vector); + ExprEvalVector vectorEval = new ExprEvalObjectVector(vector, ExpressionType.STRING); EasyMock.expect(binding.getCurrentVectorId()).andReturn(1).anyTimes(); EasyMock.expect(vectorProcessor.evalVector(binding)).andReturn(vectorEval).once(); EasyMock.replay(binding, vectorProcessor); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index b90721a46542..6efb8b4a3143 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -1203,7 +1203,6 @@ public void testArrayContainsFilterArrayStringColumns() @Test public void testArrayContainsArrayStringColumns() { - cannotVectorize(); testQuery( "SELECT ARRAY_CONTAINS(arrayStringNulls, ARRAY['a', 'b']), ARRAY_CONTAINS(arrayStringNulls, arrayString) FROM druid.arrays LIMIT 5", ImmutableList.of( @@ -1489,7 +1488,7 @@ public void testArraySliceArrayColumns() @Test public void testArrayLength() { - // Cannot vectorize due to usage of expressions. + // Cannot vectorize due to array expressions. cannotVectorize(); testQuery( @@ -1532,7 +1531,7 @@ public void testArrayLength() @Test public void testArrayLengthArrayColumn() { - // Cannot vectorize due to usage of expressions. + // Cannot vectorize due to array expressions. cannotVectorize(); testQuery( @@ -1590,7 +1589,7 @@ public void testArrayLengthArrayColumn() @Test public void testArrayAppend() { - // Cannot vectorize due to usage of expressions. + // Cannot vectorize due to array expressions. cannotVectorize(); ImmutableList results; @@ -1647,7 +1646,7 @@ public void testArrayAppend() @Test public void testArrayPrepend() { - // Cannot vectorize due to usage of expressions. + // Cannot vectorize due to array expressions. cannotVectorize(); ImmutableList results; @@ -1704,7 +1703,7 @@ public void testArrayPrepend() @Test public void testArrayPrependAppend() { - // Cannot vectorize due to usage of expressions. + // Cannot vectorize due to array expressions. cannotVectorize(); ImmutableList results; @@ -1768,7 +1767,7 @@ public void testArrayPrependAppend() @Test public void testArrayConcat() { - // Cannot vectorize due to usage of expressions. + // Cannot vectorize due to array expressions. cannotVectorize(); ImmutableList results; @@ -1825,7 +1824,7 @@ public void testArrayConcat() @Test public void testArrayOffset() { - // Cannot vectorize due to usage of expressions. + // Cannot vectorize due to array expressions. cannotVectorize(); testQuery( @@ -2105,7 +2104,7 @@ public void testArrayGroupAsFloatArray() @Test public void testArrayGroupAsArrayWithFunction() { - // Cannot vectorize due to usage of expressions. + // Cannot vectorize due to array expressions. cannotVectorize(); testQuery( "SELECT ARRAY[ARRAY_ORDINAL(dim3, 2)], SUM(cnt) FROM druid.numfoo GROUP BY 1 ORDER BY 2 DESC", @@ -2149,7 +2148,7 @@ public void testArrayGroupAsArrayWithFunction() @Test public void testArrayOrdinal() { - // Cannot vectorize due to usage of expressions. + // Cannot vectorize due to array expressions. cannotVectorize(); testQuery( @@ -2192,7 +2191,7 @@ public void testArrayOrdinal() @Test public void testArrayOffsetOf() { - // Cannot vectorize due to usage of expressions. + // Cannot vectorize due to array expressions. cannotVectorize(); testQuery( @@ -2241,7 +2240,7 @@ public void testArrayOffsetOf() @Test public void testArrayOrdinalOf() { - // Cannot vectorize due to usage of expressions. + // Cannot vectorize due to array expressions. cannotVectorize(); testQuery( @@ -2291,7 +2290,7 @@ public void testArrayOrdinalOf() @Test public void testArrayToString() { - // Cannot vectorize due to usage of expressions. + // Cannot vectorize due to array expressions. cannotVectorize(); ImmutableList results; @@ -2347,7 +2346,7 @@ public void testArrayToString() @Test public void testArrayToStringToMultiValueString() { - // Cannot vectorize due to usage of expressions. + // Cannot vectorize due to array expressions. cannotVectorize(); ImmutableList results; @@ -3446,8 +3445,6 @@ public void testArrayAggAsArrayFromJoin() @Test public void testArrayAggGroupByArrayAggFromSubquery() { - cannotVectorize(); - testQuery( "SELECT dim2, arr, COUNT(*) FROM (SELECT dim2, ARRAY_AGG(DISTINCT dim1) as arr FROM foo WHERE dim1 is not null GROUP BY 1 LIMIT 5) GROUP BY 1,2", QUERY_CONTEXT_NO_STRINGIFY_ARRAY, @@ -3877,7 +3874,6 @@ public void testArrayAggGroupByArrayContainsSubquery() @Test public void testUnnestInline() { - cannotVectorize(); testQuery( "SELECT * FROM UNNEST(ARRAY[1,2,3])", QUERY_CONTEXT_UNNEST, @@ -3943,7 +3939,6 @@ public void testUnnestInlineWithCount() @Test public void testUnnest() { - cannotVectorize(); testQuery( "SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3)", QUERY_CONTEXT_UNNEST, @@ -3988,7 +3983,6 @@ public void testUnnest() @Test public void testUnnestArrayColumnsString() { - cannotVectorize(); testQuery( "SELECT a FROM druid.arrays, UNNEST(arrayString) as unnested (a)", QUERY_CONTEXT_UNNEST, @@ -4036,7 +4030,6 @@ public void testUnnestArrayColumnsString() @Test public void testUnnestArrayColumnsStringNulls() { - cannotVectorize(); testQuery( "SELECT a FROM druid.arrays, UNNEST(arrayStringNulls) as unnested (a)", QUERY_CONTEXT_UNNEST, @@ -4083,7 +4076,6 @@ public void testUnnestArrayColumnsStringNulls() @Test public void testUnnestArrayColumnsLong() { - cannotVectorize(); testQuery( "SELECT a FROM druid.arrays, UNNEST(arrayLong) as unnested (a)", QUERY_CONTEXT_UNNEST, @@ -4137,7 +4129,6 @@ public void testUnnestArrayColumnsLong() @Test public void testUnnestArrayColumnsLongNulls() { - cannotVectorize(); testQuery( "SELECT a FROM druid.arrays, UNNEST(arrayLongNulls) as unnested (a)", QUERY_CONTEXT_UNNEST, @@ -4187,7 +4178,6 @@ public void testUnnestArrayColumnsLongNulls() @Test public void testUnnestArrayColumnsDouble() { - cannotVectorize(); testQuery( "SELECT a FROM druid.arrays, UNNEST(arrayDouble) as unnested (a)", QUERY_CONTEXT_UNNEST, @@ -4241,7 +4231,6 @@ public void testUnnestArrayColumnsDouble() @Test public void testUnnestArrayColumnsDoubleNulls() { - cannotVectorize(); testQuery( "SELECT a FROM druid.arrays, UNNEST(arrayDoubleNulls) as unnested (a)", QUERY_CONTEXT_UNNEST, @@ -4294,7 +4283,6 @@ public void testUnnestArrayColumnsDoubleNulls() @Test public void testUnnestTwice() { - cannotVectorize(); testQuery( "SELECT dim1, MV_TO_ARRAY(dim3), STRING_TO_ARRAY(dim1, U&'\\005C.') AS dim1_split, dim1_split_unnest, dim3_unnest\n" + "FROM\n" @@ -4371,7 +4359,6 @@ public void testUnnestTwice() @Test public void testUnnestTwiceArrayColumns() { - cannotVectorize(); testQuery( "SELECT arrayStringNulls, arrayLongNulls, usn, uln" + " FROM\n" @@ -4454,7 +4441,6 @@ public void testUnnestTwiceArrayColumns() @Test public void testUnnestTwiceWithFiltersAndExpressions() { - cannotVectorize(); testQuery( "SELECT dim1, MV_TO_ARRAY(dim3), STRING_TO_ARRAY(dim1, U&'\\005C.') AS dim1_split, dim1_split_unnest, dim3_unnest || 'xx'\n" + "FROM\n" @@ -4520,7 +4506,6 @@ public void testUnnestTwiceWithFiltersAndExpressions() @Test public void testUnnestThriceWithFiltersOnDimAndUnnestCol() { - cannotVectorize(); String sql = " SELECT dimZipf, dim3_unnest1, dim3_unnest2, dim3_unnest3 FROM \n" + " ( SELECT * FROM \n" + " ( SELECT * FROM lotsocolumns, UNNEST(MV_TO_ARRAY(dimMultivalEnumerated)) as ut(dim3_unnest1) )" @@ -4618,7 +4603,6 @@ public void testUnnestThriceWithFiltersOnDimAndUnnestCol() @Test public void testUnnestThriceWithFiltersOnDimAndAllUnnestColumns() { - cannotVectorize(); String sql = " SELECT dimZipf, dim3_unnest1, dim3_unnest2, dim3_unnest3 FROM \n" + " ( SELECT * FROM \n" + " ( SELECT * FROM lotsocolumns, UNNEST(MV_TO_ARRAY(dimMultivalEnumerated)) as ut(dim3_unnest1) )" @@ -4687,7 +4671,6 @@ public void testUnnestThriceWithFiltersOnDimAndAllUnnestColumns() @Test public void testUnnestThriceWithFiltersOnDimAndAllUnnestColumnsArrayColumns() { - cannotVectorize(); String sql = " SELECT arrayString, uln, udn, usn FROM \n" + " ( SELECT * FROM \n" + " ( SELECT * FROM arrays, UNNEST(arrayLongNulls) as ut(uln))" @@ -4755,7 +4738,6 @@ public void testUnnestThriceWithFiltersOnDimAndAllUnnestColumnsArrayColumns() @Test public void testUnnestThriceWithFiltersOnDimAndUnnestColumnsORCombinations() { - cannotVectorize(); String sql = " SELECT dimZipf, dim3_unnest1, dim3_unnest2, dim3_unnest3 FROM \n" + " ( SELECT * FROM \n" + " ( SELECT * FROM lotsocolumns, UNNEST(MV_TO_ARRAY(dimMultivalEnumerated)) as ut(dim3_unnest1) )" @@ -4835,7 +4817,6 @@ public void testUnnestThriceWithFiltersOnDimAndUnnestColumnsORCombinations() @Test public void testUnnestThriceWithFiltersOnDimAndAllUnnestColumnsArrayColumnsOrFilters() { - cannotVectorize(); String sql = " SELECT arrayString, uln, udn, usn FROM \n" + " ( SELECT * FROM \n" + " ( SELECT * FROM arrays, UNNEST(arrayLongNulls) as ut(uln))" @@ -5030,7 +5011,6 @@ public void testUnnestWithGroupByOrderBy() @Test public void testUnnestWithGroupByOrderByWithLimit() { - cannotVectorize(); testQuery( "SELECT d3, COUNT(*) FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) AS unnested(d3) GROUP BY d3 ORDER BY d3 ASC LIMIT 4 ", QUERY_CONTEXT_UNNEST, @@ -5106,7 +5086,6 @@ public void testUnnestWithGroupByHaving() @Test public void testUnnestWithLimit() { - cannotVectorize(); testQuery( "SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) LIMIT 3", QUERY_CONTEXT_UNNEST, @@ -5136,7 +5115,6 @@ public void testUnnestWithLimit() @Test public void testUnnestFirstQueryOnSelect() { - cannotVectorize(); testQuery( "SELECT d3 FROM (select dim1, dim2, dim3 from druid.numfoo), UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3)", QUERY_CONTEXT_UNNEST, @@ -5181,7 +5159,6 @@ public void testUnnestFirstQueryOnSelect() @Test public void testUnnestVirtualWithColumns1() { - cannotVectorize(); testQuery( "SELECT strings, m1 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (strings) where (strings='a' and (m1<=10 or strings='b'))", QUERY_CONTEXT_UNNEST, @@ -5222,7 +5199,6 @@ public void testUnnestVirtualWithColumns1() @Test public void testUnnestVirtualWithColumns2() { - cannotVectorize(); testQuery( "SELECT strings, m1 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (strings) where (strings='a' or (m1=2 and strings='b'))", QUERY_CONTEXT_UNNEST, @@ -5260,7 +5236,6 @@ public void testUnnestVirtualWithColumns2() @Test public void testUnnestWithFilters() { - cannotVectorize(); testQuery( "SELECT d3 FROM (select * from druid.numfoo where dim2='a'), UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3)", QUERY_CONTEXT_UNNEST, @@ -5292,7 +5267,6 @@ public void testUnnestWithFilters() @Test public void testUnnestWithFiltersWithExpressionInInnerQuery() { - cannotVectorize(); testQuery( "SELECT t,d3 FROM (select FLOOR(__time to hour) t, dim3 from druid.numfoo where dim2='a'), UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3)", QUERY_CONTEXT_UNNEST, @@ -5329,7 +5303,6 @@ public void testUnnestWithFiltersWithExpressionInInnerQuery() @Test public void testUnnestWithInFiltersWithExpressionInInnerQuery() { - cannotVectorize(); testQuery( "SELECT t,d3 FROM (select FLOOR(__time to hour) t, dim3 from druid.numfoo where dim2 IN ('a','b')), UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3)", QUERY_CONTEXT_UNNEST, @@ -5364,7 +5337,6 @@ public void testUnnestWithInFiltersWithExpressionInInnerQuery() @Test public void testUnnestWithFiltersInnerLimit() { - cannotVectorize(); testQuery( "SELECT d3 FROM (select dim2,dim3 from druid.numfoo where dim2='a' LIMIT 2), UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3)", QUERY_CONTEXT_UNNEST, @@ -5411,7 +5383,6 @@ public void testUnnestWithFiltersInnerLimit() @Test public void testUnnestWithFiltersInsideAndOutside() { - cannotVectorize(); testQuery( "SELECT d3 FROM\n" + " (select * from druid.numfoo where dim2='a') as t,\n" @@ -5449,7 +5420,6 @@ public void testUnnestWithFiltersInsideAndOutside() @Test public void testUnnestWithFiltersInsideAndOutside1() { - cannotVectorize(); testQuery( "SELECT d3 FROM\n" + " (select * from druid.numfoo where dim2='a'),\n" @@ -5490,7 +5460,6 @@ public void testUnnestWithFiltersInsideAndOutside1() @Test public void testUnnestWithFiltersOutside() { - cannotVectorize(); testQuery( "SELECT d3 FROM\n" + " druid.numfoo t,\n" @@ -5532,7 +5501,6 @@ public void testUnnestWithFiltersOutside() @Test public void testUnnestWithInFilters() { - cannotVectorize(); testQuery( "SELECT d3 FROM (select * from druid.numfoo where dim2 IN ('a','b','ab','abc')), UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3)", QUERY_CONTEXT_UNNEST, @@ -5566,7 +5534,6 @@ public void testUnnestWithInFilters() @Test public void testUnnestVirtualWithColumns() { - cannotVectorize(); testQuery( "SELECT strings FROM druid.numfoo, UNNEST(ARRAY[dim4, dim5]) as unnested (strings)", QUERY_CONTEXT_UNNEST, @@ -5659,7 +5626,6 @@ public void testUnnestWithGroupByOrderByOnVirtualColumn() @Test public void testUnnestWithJoinOnTheLeft() { - cannotVectorize(); testQuery( "SELECT d3 from (SELECT * from druid.numfoo JOIN (select dim2 as t from druid.numfoo where dim2 IN ('a','b','ab','abc')) ON dim2=t), UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3)", QUERY_CONTEXT_UNNEST, @@ -5723,7 +5689,6 @@ public void testUnnestWithConstant() // Since there is a constant on the right, // Druid will plan this as a join query // as there is nothing to correlate between left and right - cannotVectorize(); testQuery( "SELECT longs FROM druid.numfoo, UNNEST(ARRAY[1,2,3]) as unnested (longs)", QUERY_CONTEXT_UNNEST, @@ -5778,7 +5743,6 @@ public void testUnnestWithConstant() @Test public void testUnnestWithSQLFunctionOnUnnestedColumn() { - cannotVectorize(); testQuery( "SELECT strlen(d3) FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3)", QUERY_CONTEXT_UNNEST, @@ -5824,7 +5788,6 @@ public void testUnnestWithSQLFunctionOnUnnestedColumn() @Test public void testUnnestWithINFiltersWithLeftRewrite() { - cannotVectorize(); testQuery( "SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) where d3 IN ('a','b') and m1 < 10", QUERY_CONTEXT_UNNEST, @@ -5856,7 +5819,6 @@ public void testUnnestWithINFiltersWithLeftRewrite() @Test public void testUnnestWithINFiltersWithNoLeftRewrite() { - cannotVectorize(); testQuery( "SELECT d45 FROM druid.numfoo, UNNEST(ARRAY[dim4,dim5]) as unnested (d45) where d45 IN ('a','b')", QUERY_CONTEXT_UNNEST, @@ -5888,7 +5850,6 @@ public void testUnnestWithINFiltersWithNoLeftRewrite() @Test public void testUnnestWithInvalidINFiltersOnUnnestedColumn() { - cannotVectorize(); testQuery( "SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) where d3 IN ('foo','bar')", QUERY_CONTEXT_UNNEST, @@ -5913,7 +5874,6 @@ public void testUnnestWithInvalidINFiltersOnUnnestedColumn() @Test public void testUnnestWithNotFiltersOnUnnestedColumn() { - cannotVectorize(); testQuery( "SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) where d3!='d' ", QUERY_CONTEXT_UNNEST, @@ -5954,7 +5914,6 @@ public void testUnnestWithNotFiltersOnUnnestedColumn() @Test public void testUnnestWithSelectorFiltersOnSelectedColumn() { - cannotVectorize(); testQuery( "SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) where d3='b'", QUERY_CONTEXT_UNNEST, @@ -5982,7 +5941,6 @@ public void testUnnestWithSelectorFiltersOnSelectedColumn() @Test public void testUnnestWithSelectorFiltersOnVirtualColumn() { - cannotVectorize(); testQuery( "SELECT d12 FROM druid.numfoo, UNNEST(ARRAY[m1,m2]) as unnested (d12) where d12=1", QUERY_CONTEXT_UNNEST, @@ -6012,7 +5970,6 @@ public void testUnnestWithSelectorFiltersOnVirtualColumn() @Test public void testUnnestWithSelectorFiltersOnVirtualStringColumn() { - cannotVectorize(); testQuery( "SELECT d45 FROM druid.numfoo, UNNEST(ARRAY[dim4,dim5]) as unnested (d45) where d45 IN ('a','ab')", QUERY_CONTEXT_UNNEST, @@ -6043,7 +6000,6 @@ public void testUnnestWithSelectorFiltersOnVirtualStringColumn() @Test public void testUnnestWithMultipleAndFiltersOnSelectedColumns() { - cannotVectorize(); testQuery( "SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) where d3='b' and m1 < 10 and m2 < 10", QUERY_CONTEXT_UNNEST, @@ -6077,7 +6033,6 @@ public void testUnnestWithMultipleAndFiltersOnSelectedColumns() @Test public void testUnnestWithMultipleOrFiltersOnSelectedColumns() { - cannotVectorize(); testQuery( "SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) where d3='b' or m1 < 2 ", QUERY_CONTEXT_UNNEST, @@ -6112,7 +6067,6 @@ public void testUnnestWithMultipleOrFiltersOnSelectedColumns() @Test public void testUnnestWithMultipleAndFiltersOnSelectedUnnestedColumns() { - cannotVectorize(); testQuery( "SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) where d3 IN ('a','b') and d3 < 'e' ", QUERY_CONTEXT_UNNEST, @@ -6141,7 +6095,6 @@ public void testUnnestWithMultipleAndFiltersOnSelectedUnnestedColumns() @Test public void testUnnestWithMultipleOrFiltersOnUnnestedColumns() { - cannotVectorize(); testQuery( "SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) where d3='b' or d3='d' ", QUERY_CONTEXT_UNNEST, @@ -6170,7 +6123,6 @@ public void testUnnestWithMultipleOrFiltersOnUnnestedColumns() @Test public void testUnnestWithMultipleOrFiltersOnVariationsOfUnnestedColumns() { - cannotVectorize(); testQuery( "SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) where strlen(d3) < 2 or d3='d' ", QUERY_CONTEXT_UNNEST, @@ -6216,7 +6168,6 @@ public void testUnnestWithMultipleOrFiltersOnVariationsOfUnnestedColumns() @Test public void testUnnestWithMultipleOrFiltersOnSelectedNonUnnestedColumns() { - cannotVectorize(); testQuery( "SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) where m1 < 2 or m2 < 2 ", QUERY_CONTEXT_UNNEST, @@ -6250,7 +6201,6 @@ public void testUnnestWithMultipleOrFiltersOnSelectedNonUnnestedColumns() @Test public void testUnnestWithMultipleOrFiltersOnSelectedVirtualColumns() { - cannotVectorize(); testQuery( "SELECT d45 FROM druid.numfoo, UNNEST(ARRAY[dim4,dim5]) as unnested (d45) where d45 IN ('a','aa') or m1 < 2 ", QUERY_CONTEXT_UNNEST, @@ -6287,7 +6237,6 @@ public void testUnnestWithMultipleOrFiltersOnSelectedVirtualColumns() @Test public void testUnnestWithMultipleOrFiltersOnUnnestedColumnsAndOnOriginalColumn() { - cannotVectorize(); testQuery( "SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) where d3='b' or dim3='d' ", QUERY_CONTEXT_UNNEST, @@ -6322,7 +6271,6 @@ public void testUnnestWithMultipleOrFiltersOnUnnestedColumnsAndOnOriginalColumn( @Test public void testUnnestWithMultipleOrFiltersOnUnnestedColumnsAndOnOriginalColumnDiffOrdering() { - cannotVectorize(); testQuery( "SELECT dim3, d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) where dim3='b' or d3='a' ", QUERY_CONTEXT_UNNEST, @@ -6610,7 +6558,6 @@ public void testUnnestWithGroupByHavingWithWhereOnUnnestArrayCol() @Test public void testUnnestVirtualWithColumnsAndNullIf() { - cannotVectorize(); testQuery( "select c,m2 from druid.foo, unnest(ARRAY[\"m1\", \"m2\"]) as u(c) where NULLIF(c,m2) IS NULL", QUERY_CONTEXT_UNNEST, @@ -7254,7 +7201,6 @@ public void testArrayToMvPostaggInline() @Test public void testUnnestExtractionFn() { - cannotVectorize(); testQuery( "SELECT substring(d3,1) FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) WHERE substring(d3,1) <> 'b'", QUERY_CONTEXT_UNNEST, @@ -7295,7 +7241,6 @@ public void testUnnestExtractionFn() @Test public void testUnnestExtractionFnNull() { - cannotVectorize(); testQuery( "SELECT substring(d3,1) FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) WHERE substring(d3,1) is not null", QUERY_CONTEXT_UNNEST, diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java index b4ebf1ecf686..ce9aa2c7a621 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java @@ -4224,9 +4224,6 @@ public void testSemiAndAntiJoinSimultaneouslyUsingExplicitJoins(Map queryContext) { // Native JOIN operator cannot handle the condition, so a SQL JOIN with greater-than is translated into a // cross join with a filter. - cannotVectorize(); // We don't handle non-equi join conditions for non-sql compatible mode. Assumptions.assumeFalse(NullHandling.replaceWithDefault()); @@ -4760,7 +4756,6 @@ public void testJoinWithEquiAndNonEquiCondition(Map queryContext { // Native JOIN operator cannot handle the condition, so a SQL JOIN with greater-than is translated into a // cross join with a filter. - cannotVectorize(); // We don't handle non-equi join conditions for non-sql compatible mode. Assumptions.assumeFalse(NullHandling.replaceWithDefault()); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteLookupFunctionQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteLookupFunctionQueryTest.java index 45298d9c513d..292d2637559a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteLookupFunctionQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteLookupFunctionQueryTest.java @@ -84,8 +84,6 @@ public class CalciteLookupFunctionQueryTest extends BaseCalciteQueryTest @Test public void testFilterEquals() { - cannotVectorize(); - testQuery( buildFilterTestSql("LOOKUP(dim1, 'lookyloo') = 'xabc'"), QUERY_CONTEXT, @@ -129,8 +127,6 @@ public void testFilterFunctionOfLookup() @Test public void testFilterLookupOfConcat() { - cannotVectorize(); - testQuery( buildFilterTestSql("LOOKUP(CONCAT(dim1, 'b', dim2), 'lookyloo') = 'xabc'"), QUERY_CONTEXT, @@ -172,8 +168,6 @@ public void testFilterInLookupOfConcat() @Test public void testFilterConcatOfLookup() { - cannotVectorize(); - testQuery( buildFilterTestSql("CONCAT(LOOKUP(dim1, 'lookyloo'), ' (', dim1, ')') = 'xabc (abc)'"), QUERY_CONTEXT, @@ -211,8 +205,6 @@ public void testFilterInConcatOfLookup() @Test public void testFilterConcatOfLookupOfConcat() { - cannotVectorize(); - testQuery( buildFilterTestSql( "CONCAT(LOOKUP(CONCAT(dim1, 'b', dim2), 'lookyloo'), ' (', CONCAT(dim1, 'b', dim2), ')') = 'xabc (abc)'"), @@ -267,8 +259,6 @@ public void testFilterInConcatOfLookupOfConcat() @Test public void testFilterConcatOfCoalesceLookupOfConcat() { - cannotVectorize(); - testQuery( buildFilterTestSql( "CONCAT(COALESCE(LOOKUP(CONCAT(dim1, 'b', dim2), 'lookyloo'), 'N/A'), ' (', CONCAT(dim1, 'b', dim2), ')') = 'xabc (abc)'"), @@ -287,8 +277,6 @@ public void testFilterConcatOfCoalesceLookupOfConcat() @Test public void testFilterImpossibleLookupOfConcat() { - cannotVectorize(); - // No keys in the lookup table begin with 'key:', so this is always false. testQuery( buildFilterTestSql("LOOKUP('key:' || dim1, 'lookyloo') = 'xabc'"), @@ -301,8 +289,6 @@ public void testFilterImpossibleLookupOfConcat() @Test public void testFilterChainedEquals() { - cannotVectorize(); - testQuery( buildFilterTestSql("LOOKUP(LOOKUP(dim1, 'lookyloo'), 'lookyloo-chain') = 'zabc'"), QUERY_CONTEXT, @@ -314,8 +300,6 @@ public void testFilterChainedEquals() @Test public void testFilterEqualsLiteralFirst() { - cannotVectorize(); - testQuery( buildFilterTestSql("'xabc' = LOOKUP(dim1, 'lookyloo')"), QUERY_CONTEXT, @@ -327,8 +311,6 @@ public void testFilterEqualsLiteralFirst() @Test public void testFilterEqualsAlwaysFalse() { - cannotVectorize(); - testQuery( buildFilterTestSql("LOOKUP(dim1, 'lookyloo') = 'nonexistent'"), QUERY_CONTEXT, @@ -340,8 +322,6 @@ public void testFilterEqualsAlwaysFalse() @Test public void testFilterIsNotDistinctFrom() { - cannotVectorize(); - testQuery( buildFilterTestSql("LOOKUP(dim1, 'lookyloo') IS NOT DISTINCT FROM 'xabc'"), QUERY_CONTEXT, @@ -1392,8 +1372,6 @@ public void testFilterCoalesceSameLiteralNotEqualsInjective() @Test public void testFilterCoalesceDifferentLiteral() { - cannotVectorize(); - testQuery( buildFilterTestSql("COALESCE(LOOKUP(dim1, 'lookyloo'), 'xyzzy') = 'x6'"), QUERY_CONTEXT, @@ -1405,8 +1383,6 @@ public void testFilterCoalesceDifferentLiteral() @Test public void testFilterCoalesceDifferentLiteralAlwaysFalse() { - cannotVectorize(); - testQuery( buildFilterTestSql("COALESCE(LOOKUP(dim1, 'lookyloo'), 'xyzzy') = 'nonexistent'"), QUERY_CONTEXT, @@ -1418,8 +1394,6 @@ public void testFilterCoalesceDifferentLiteralAlwaysFalse() @Test public void testFilterCoalesceCastVarcharDifferentLiteral() { - cannotVectorize(); - testQuery( buildFilterTestSql("COALESCE(CAST(LOOKUP(dim1, 'lookyloo') AS VARCHAR), 'xyzzy') = 'x6'"), QUERY_CONTEXT, @@ -1895,8 +1869,6 @@ public void testPullUpAndReverseLookup() @Test public void testDontPullUpLookupWhenUsedByAggregation() { - cannotVectorize(); - testQuery( "SELECT LOOKUP(dim1, 'lookyloo121'), COUNT(LOOKUP(dim1, 'lookyloo121')) FROM druid.foo GROUP BY 1", ImmutableList.of( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java index 3ec994c2e3ee..275b4e9a164f 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java @@ -1122,9 +1122,6 @@ public void testMultiValueStringToStringToMultiValueString() @Test public void testSelectAndFilterByStringToMV() { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - testBuilder() .sql("SELECT STRING_TO_MV(CONCAT(MV_TO_STRING(dim3, ','), ',d'), ',') FROM druid.numfoo " + "WHERE MV_CONTAINS(STRING_TO_MV(CONCAT(MV_TO_STRING(dim3, ','), ',d'), ','), 'd')") @@ -1167,9 +1164,6 @@ public void testSelectAndFilterByStringToMV() @Test public void testStringToMVOfConstant() { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - testBuilder() .sql("SELECT m1, STRING_TO_MV('a,b', ',') AS mv FROM druid.numfoo GROUP BY 1") .expectedQuery( @@ -1339,9 +1333,6 @@ public void testMultiValueListFilter() @Test public void testMultiValueListFilterNonLiteral() { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - testQuery( "SELECT MV_FILTER_ONLY(dim3, ARRAY[dim2]) FROM druid.numfoo", ImmutableList.of( @@ -1429,9 +1420,6 @@ public void testMultiValueListFilterDeny() @Test public void testMultiValueListFilterDenyNonLiteral() { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - testQuery( "SELECT MV_FILTER_NONE(dim3, ARRAY[dim2]) FROM druid.numfoo", ImmutableList.of( @@ -2170,7 +2158,6 @@ public void testMultiValueToArrayArgsWithArray() @Test public void testMultiValueStringOverlapFilterCoalesceNvl() { - cannotVectorize(); testQuery( "SELECT COALESCE(dim3, 'other') FROM druid.numfoo " + "WHERE MV_OVERLAP(COALESCE(MV_TO_ARRAY(dim3), ARRAY['other']), ARRAY['a', 'b', 'other']) OR " @@ -2392,7 +2379,6 @@ public void testMvContainsFilterWithExtractionFn() @Test public void testMvContainsSelectColumns() { - cannotVectorize(); testQuery( "SELECT MV_CONTAINS(dim3, ARRAY['a', 'b']), MV_OVERLAP(dim3, ARRAY['a', 'b']) FROM druid.numfoo LIMIT 5", ImmutableList.of( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java index eda86289d29d..f8b37a435087 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java @@ -1276,7 +1276,6 @@ public void testUnnestRootSingleTypeArrayLongNulls() @Test public void testUnnestRootSingleTypeArrayStringNulls() { - cannotVectorize(); testBuilder() .sql("SELECT strings FROM druid.arrays, UNNEST(arrayStringNulls) as u(strings)") .queryContext(QUERY_CONTEXT_NO_STRINGIFY_ARRAY) @@ -1334,7 +1333,6 @@ public void testUnnestRootSingleTypeArrayStringNulls() @Test public void testUnnestRootSingleTypeArrayDoubleNulls() { - cannotVectorize(); testBuilder() .sql("SELECT doubles FROM druid.arrays, UNNEST(arrayDoubleNulls) as u(doubles)") .queryContext(QUERY_CONTEXT_NO_STRINGIFY_ARRAY) @@ -2242,7 +2240,6 @@ public void testGroupByRootSingleTypeArrayDoubleNullsFiltered() @Test public void testGroupByRootSingleTypeArrayLongElement() { - cannotVectorize(); testBuilder() .sql( "SELECT " @@ -2290,7 +2287,6 @@ public void testGroupByRootSingleTypeArrayLongElement() @Test public void testGroupByRootSingleTypeArrayLongElementFiltered() { - cannotVectorize(); testBuilder() .sql( "SELECT " @@ -2338,7 +2334,6 @@ public void testGroupByRootSingleTypeArrayLongElementFiltered() @Test public void testGroupByRootSingleTypeArrayLongElementDefault() { - cannotVectorize(); testBuilder() .sql( "SELECT " @@ -2654,7 +2649,6 @@ public void testGroupByPathSelectorFilter() @Test public void testGroupByPathSelectorFilterCoalesce() { - cannotVectorize(); testQuery( "SELECT " + "JSON_VALUE(nest, '$.x'), " @@ -6756,7 +6750,6 @@ public void testGroupByPathDynamicArg() @Test public void testJsonQueryDynamicArg() { - cannotVectorize(); testQuery( "SELECT JSON_PATHS(nester), JSON_QUERY(nester, ARRAY_OFFSET(JSON_PATHS(nester), 0))\n" + "FROM druid.nested", @@ -6801,7 +6794,6 @@ public void testJsonQueryDynamicArg() @Test public void testJsonQueryArrays() { - cannotVectorize(); testBuilder() .sql("SELECT JSON_QUERY_ARRAY(arrayObject, '$') FROM druid.arrays") .queryContext(QUERY_CONTEXT_DEFAULT) @@ -6853,7 +6845,6 @@ public void testJsonQueryArrays() @Test public void testJsonQueryArrayNullArray() { - cannotVectorize(); testBuilder() .sql("SELECT JSON_QUERY_ARRAY(arrayObject, '$.') FROM druid.arrays where arrayObject is null limit 1") .queryContext(QUERY_CONTEXT_DEFAULT) @@ -6893,7 +6884,6 @@ public void testJsonQueryArrayNullArray() @Test public void testUnnestJsonQueryArrays() { - cannotVectorize(); testBuilder() .sql("SELECT objects FROM druid.arrays, UNNEST(JSON_QUERY_ARRAY(arrayObject, '$')) as u(objects)") .queryContext(QUERY_CONTEXT_NO_STRINGIFY_ARRAY) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteParameterQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteParameterQueryTest.java index 3a35295d0d27..6bd273696e2a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteParameterQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteParameterQueryTest.java @@ -652,10 +652,6 @@ public void testPartiallyMissingParameterInTheMiddle() @Test public void testWrongTypeParameter() { - if (!useDefault) { - // cannot vectorize inline datasource - cannotVectorize(); - } testQuery( "SELECT COUNT(*)\n" + "FROM druid.numfoo\n" @@ -701,7 +697,6 @@ public void testWrongTypeParameter() @Test public void testNullParameter() { - cannotVectorize(); // contrived example of using null as an sql parameter to at least test the codepath because lots of things dont // actually work as null and things like 'IS NULL' fail to parse in calcite if expressed as 'IS ?' diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index cc97e967d5f7..6dbe93196a1b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -582,7 +582,6 @@ public void testGroupByWithForceLimitPushDown() @Test public void testSafeDivide() { - cannotVectorize(); final Map context = new HashMap<>(QUERY_CONTEXT_DEFAULT); testQuery( @@ -2604,11 +2603,6 @@ public void testExactCountDistinctWithFilter() ) ); - if (NullHandling.sqlCompatible()) { - // Cannot vectorize due to "istrue" operator. - cannotVectorize(); - } - testQuery( PLANNER_CONFIG_NO_HLL.withOverrides( ImmutableMap.of( @@ -3096,9 +3090,6 @@ public void testPruneDeadAggregatorsThroughHaving() @Test public void testGroupByCaseWhen() { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT\n" + " CASE EXTRACT(DAY FROM __time)\n" @@ -3149,9 +3140,6 @@ public void testGroupByCaseWhen() @Test public void testDecomposeCaseWhenThreeArg() { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT\n" + " dim1, dim2, CASE WHEN dim1 = 'abc' THEN dim1 ELSE dim2 END\n" @@ -3193,9 +3181,6 @@ public void testDecomposeCaseWhenThreeArg() @Test public void testDecomposeCaseWhenTwoArg() { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT\n" + " dim1, dim2, CASE WHEN dim1 = 'def' THEN dim2 END\n" @@ -3228,9 +3213,6 @@ public void testDecomposeCaseWhenTwoArg() @Test public void testGroupByCaseWhenOfTripleAnd() { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT\n" + " CASE WHEN m1 > 1 AND m1 < 5 AND cnt = 1 THEN 'x' ELSE NULL END," @@ -3827,10 +3809,6 @@ public void testCoalesceColumns() { // Doesn't conform to the SQL standard, but it's how we do it. // This example is used in the sql.md doc. - - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT COALESCE(dim2, dim1), COUNT(*) FROM druid.foo GROUP BY COALESCE(dim2, dim1)\n", ImmutableList.of( @@ -3869,9 +3847,6 @@ public void testCoalesceColumns() @Test public void testCoalesceColumnsFilter() { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT COALESCE(dim2, dim1), COUNT(*) FROM druid.foo WHERE COALESCE(dim2, dim1) IN ('a', 'abc') GROUP BY COALESCE(dim2, dim1)\n", ImmutableList.of( @@ -3905,9 +3880,6 @@ public void testCoalesceColumnsFilter() @Test public void testCoalesceColumnsFilterWithEquality() { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - // we can remove this test if PlannerContext.CTX_SQL_USE_BOUNDS_AND_SELECTORS ever defaults to false all the time // since it otherwise is a duplicate of testCoalesceColumnsFilter @@ -4315,10 +4287,6 @@ public void testCountStar() @Test public void testCountStarOnCommonTableExpression() { - if (NullHandling.sqlCompatible()) { - // cannot vectorize due to substring expression - cannotVectorize(); - } Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) @@ -4356,11 +4324,6 @@ public void testCountStarOnCommonTableExpression() @Test public void testCountStarOnView() { - if (NullHandling.sqlCompatible()) { - // cannot vectorize due to substring expression - cannotVectorize(); - } - Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) @@ -4397,10 +4360,6 @@ public void testCountStarOnView() @Test public void testConfusedView() { - if (NullHandling.sqlCompatible()) { - // cannot vectorize due to substring expression - cannotVectorize(); - } Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) @@ -4991,10 +4950,6 @@ public void testGroupByWithSortOnPostAggregationNoTopNContext() @Test public void testFilteredAggregations() { - if (NullHandling.sqlCompatible()) { - // cannot vectorize due to expression filter - cannotVectorize(); - } Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) @@ -5244,9 +5199,6 @@ public void testFilteredAggregationWithNotIn() @Test public void testExpressionAggregations() { - // Cannot vectorize due to expressions. - cannotVectorize(); - final ExprMacroTable macroTable = CalciteTests.createExprMacroTable(); testQuery( @@ -5739,9 +5691,6 @@ public void testCountStarWithDegenerateFilter() public void testCountStarWithNotOfDegenerateFilter() { msqIncompatible(); - // HashJoinSegmentStorageAdapter is not vectorizable - cannotVectorize(); - testQuery( "SELECT COUNT(*) FROM druid.foo WHERE dim2 = 'a' and not (dim1 > 'a' OR dim1 < 'b')", ImmutableList.of( @@ -6689,9 +6638,6 @@ public void testSumOfString() @Test public void testSumOfExtractionFn() { - // Cannot vectorize due to expressions in aggregators. - cannotVectorize(); - testQuery( "SELECT SUM(CAST(SUBSTRING(dim1, 1, 10) AS INTEGER)) FROM druid.foo", ImmutableList.of( @@ -7211,7 +7157,8 @@ public void testMultipleExactCountDistinctWithGroupingUsingGroupingSets() public void testApproxCountDistinct() { msqIncompatible(); - // Cannot vectorize due to virtual columns. + + // Cannot vectorize due to multi-valued dim2. cannotVectorize(); testQuery( @@ -7986,9 +7933,6 @@ public void testCountDistinctOfSubstring() public void testCountDistinctOfTrim() { // Test a couple different syntax variants of TRIM. - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT COUNT(DISTINCT TRIM(BOTH ' ' FROM dim1)) FROM druid.foo WHERE TRIM(dim1) <> ''", ImmutableList.of( @@ -8022,10 +7966,6 @@ public void testCountDistinctOfTrim() public void testSillyQuarters() { // Like FLOOR(__time TO QUARTER) but silly. - - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT CAST((EXTRACT(MONTH FROM __time) - 1 ) / 3 + 1 AS INTEGER) AS quarter, COUNT(*)\n" + "FROM foo\n" @@ -8135,8 +8075,6 @@ public void testRegexpExtractWithBadRegexPattern() @Test public void testRegexpExtractFilterViaNotNullCheck() { - // Cannot vectorize due to extractionFn in dimension spec. - cannotVectorize(); Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) @@ -8312,7 +8250,8 @@ public void testGroupByLimitPushDownWithHavingOnLong() @Test public void testGroupByLimitPushdownExtraction() { - cannotVectorize(); + // Skip vectorization because this can vectorize with decoupled planning, but cannot with regular planning. + skipVectorize(); testQuery( "SELECT dim4, substring(dim5, 1, 1), count(*) FROM druid.numfoo WHERE dim4 = 'a' GROUP BY 1,2 LIMIT 2", @@ -8570,9 +8509,6 @@ public void testFilterOnTimeFloorComparisonMisaligned() @Test public void testFilterOnTimeExtract() { - // Cannot vectorize due to expression filter. - cannotVectorize(); - testQuery( "SELECT COUNT(*) FROM druid.foo\n" + "WHERE EXTRACT(YEAR FROM __time) = 2000\n" @@ -8605,9 +8541,6 @@ public void testFilterOnTimeExtract() @Test public void testFilterOnTimeExtractWithMultipleDays() { - // Cannot vectorize due to expression filters. - cannotVectorize(); - testQuery( "SELECT COUNT(*) FROM druid.foo\n" + "WHERE EXTRACT(YEAR FROM __time) = 2000\n" @@ -8649,9 +8582,6 @@ public void testFilterOnTimeExtractWithMultipleDays() public void testFilterOnTimeExtractWithVariousTimeUnits() { msqIncompatible(); - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT COUNT(*) FROM druid.foo4\n" + "WHERE EXTRACT(YEAR FROM __time) = 2000\n" @@ -8761,7 +8691,6 @@ public void testGroupByFloor() public void testQueryWithSelectProjectAndIdentityProjectDoesNotRename() { msqIncompatible(); - cannotVectorize(); testQuery( PLANNER_CONFIG_NO_HLL.withOverrides( ImmutableMap.of( @@ -9043,9 +8972,6 @@ public void testGroupByFloorTimeAndOneOtherDimensionWithOrderBy() @Test public void testGroupByStringLength() { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT CHARACTER_LENGTH(dim1), COUNT(*) FROM druid.foo GROUP BY CHARACTER_LENGTH(dim1)", ImmutableList.of( @@ -9287,9 +9213,6 @@ public void testTimeseriesUsingTimeFloor() @Test public void testTimeseriesUsingTimeFloorWithTimeShift() { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT SUM(cnt), gran FROM (\n" + " SELECT TIME_FLOOR(TIME_SHIFT(__time, 'P1D', -1), 'P1M') AS gran,\n" @@ -10242,9 +10165,6 @@ public void testGroupByAggregatorDefaultValuesNonVectorized() @Test public void testGroupByExtractYear() { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT\n" + " EXTRACT(YEAR FROM __time) AS \"year\",\n" @@ -10293,9 +10213,6 @@ public void testGroupByExtractYear() @Test public void testGroupByFormatYearAndMonth() { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT\n" + " TIME_FORMAt(__time, 'yyyy MM') AS \"year\",\n" @@ -10344,9 +10261,6 @@ public void testGroupByFormatYearAndMonth() @Test public void testGroupByExtractFloorTime() { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT\n" + "EXTRACT(YEAR FROM FLOOR(__time TO YEAR)) AS \"year\", SUM(cnt)\n" @@ -10379,9 +10293,6 @@ public void testGroupByExtractFloorTime() @Test public void testGroupByExtractFloorTimeLosAngeles() { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( PLANNER_CONFIG_DEFAULT, QUERY_CONTEXT_LOS_ANGELES, @@ -10705,9 +10616,6 @@ public void testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim() public void testGroupingSets() { msqIncompatible(); - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT dim2, gran, SUM(cnt), GROUPING(dim2, gran)\n" + "FROM (SELECT FLOOR(__time TO MONTH) AS gran, COALESCE(dim2, '') dim2, cnt FROM druid.foo) AS x\n" @@ -10925,9 +10833,6 @@ public void testGroupingSetsWithNumericDimension() public void testGroupByRollup() { msqIncompatible(); - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT dim2, gran, SUM(cnt)\n" + "FROM (SELECT FLOOR(__time TO MONTH) AS gran, COALESCE(dim2, '') dim2, cnt FROM druid.foo) AS x\n" @@ -11041,9 +10946,6 @@ public void testGroupByRollupDifferentOrder() public void testGroupByCube() { msqIncompatible(); - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT dim2, gran, SUM(cnt)\n" + "FROM (SELECT FLOOR(__time TO MONTH) AS gran, COALESCE(dim2, '') dim2, cnt FROM druid.foo) AS x\n" @@ -11104,9 +11006,6 @@ public void testGroupByCube() public void testGroupingSetsWithDummyDimension() { msqIncompatible(); - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT dim2, gran, SUM(cnt)\n" + "FROM (SELECT FLOOR(__time TO MONTH) AS gran, COALESCE(dim2, '') dim2, cnt FROM druid.foo) AS x\n" @@ -11167,9 +11066,6 @@ public void testGroupingSetsWithDummyDimension() public void testGroupingSetsNoSuperset() { msqIncompatible(); - // Cannot vectorize due to virtual columns. - cannotVectorize(); - // Note: the grouping sets are reordered in the output of this query, but this is allowed. testQuery( "SELECT dim2, gran, SUM(cnt)\n" @@ -11297,9 +11193,6 @@ public void testGroupingSetsWithOrderByDimension() public void testGroupingSetsWithOrderByAggregator() { msqIncompatible(); - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT dim2, gran, SUM(cnt)\n" + "FROM (SELECT FLOOR(__time TO MONTH) AS gran, COALESCE(dim2, '') dim2, cnt FROM druid.foo) AS x\n" @@ -11367,9 +11260,6 @@ public void testGroupingSetsWithOrderByAggregator() public void testGroupingSetsWithOrderByAggregatorWithLimit() { msqIncompatible(); - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT dim2, gran, SUM(cnt)\n" + "FROM (SELECT FLOOR(__time TO MONTH) AS gran, COALESCE(dim2, '') dim2, cnt FROM druid.foo) AS x\n" @@ -13038,8 +12928,6 @@ public void testTimeStampAddZeroMonthPeriod() @Test public void testTimeStampAddZeroYearPeriod() { - cannotVectorize(); - testQuery( "SELECT TIMESTAMPADD(YEAR, 0, \"__time\") FROM druid.foo", @@ -13204,9 +13092,6 @@ public void testGroupingSetsWithLimit() public void testGroupingSetsWithLimitOrderByGran() { msqIncompatible(); - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT dim2, gran, SUM(cnt)\n" + "FROM (SELECT FLOOR(__time TO MONTH) AS gran, COALESCE(dim2, '') dim2, cnt FROM druid.foo) AS x\n" @@ -13390,7 +13275,6 @@ public void testCountAndAverageByConstantVirtualColumn() @Test public void testExpressionCounts() { - cannotVectorize(); testQuery( "SELECT\n" + " COUNT(reverse(dim2)),\n" @@ -14250,7 +14134,6 @@ public void testCommonVirtualExpressionWithDifferentValueType() @Test public void testReturnEmptyRowWhenGroupByIsConvertedToTimeseriesWithSingleConstantDimension() { - cannotVectorize(); testQuery( "SELECT 'A' from foo WHERE m1 = 50 AND dim1 = 'wat' GROUP BY 'foobar'", ImmutableList.of( @@ -14304,7 +14187,6 @@ public void testReturnEmptyRowWhenGroupByIsConvertedToTimeseriesWithSingleConsta @Test public void testReturnEmptyRowWhenGroupByIsConvertedToTimeseriesWithMultipleConstantDimensions() { - cannotVectorize(); testQuery( "SELECT 'A', dim1 from foo WHERE m1 = 50 AND dim1 = 'wat' GROUP BY dim1", ImmutableList.of( @@ -14415,7 +14297,6 @@ public void testGreatestFunctionForNumberWithIsNull() .build() ); } else { - cannotVectorize(); expectedResult = ImmutableList.of( new Object[]{"", false}, new Object[]{"1", true}, @@ -14456,7 +14337,6 @@ public void testGreatestFunctionForNumberWithIsNull() public void testGreatestFunctionForStringWithIsNull() { msqIncompatible(); - cannotVectorize(); String query = "SELECT l1, LATEST(GREATEST(dim1, dim2)) IS NULL FROM druid.numfoo GROUP BY l1"; @@ -14603,7 +14483,6 @@ public void testTimeseriesQueryWithEmptyInlineDatasourceAndGranularity() @Test public void testComplexDecode() { - cannotVectorize(); for (String complexDecode : Arrays.asList("COMPLEX_DECODE_BASE64", "DECODE_BASE64_COMPLEX")) { testQuery( StringUtils.format( @@ -14643,7 +14522,6 @@ public void testComplexDecode() public void testComplexDecodeAgg() { msqIncompatible(); - cannotVectorize(); testQuery( "SELECT APPROX_COUNT_DISTINCT_BUILTIN(COMPLEX_DECODE_BASE64('hyperUnique',PARSE_JSON(TO_JSON_STRING(unique_dim1)))) from druid.foo", ImmutableList.of( @@ -14677,7 +14555,6 @@ public void testComplexDecodeAgg() public void testComplexDecodeAggWithCastedTypeName() { msqIncompatible(); - cannotVectorize(); testQuery( "SELECT " + "APPROX_COUNT_DISTINCT_BUILTIN(COMPLEX_DECODE_BASE64(CAST('hyperUnique' AS VARCHAR),PARSE_JSON(TO_JSON_STRING(unique_dim1)))) " @@ -14925,7 +14802,6 @@ public void testGroupByDateTrunc() public void testLatestByOnStringColumnWithoutMaxBytesSpecified() { String defaultString = useDefault ? "" : null; - cannotVectorize(); testQuery( "SELECT dim2,LATEST(dim3),LATEST_BY(dim1, __time),EARLIEST(dim3),EARLIEST_BY(dim1, __time),ANY_VALUE(dim3) FROM druid.foo where dim2='abc' group by 1", ImmutableList.of( @@ -15048,8 +14924,6 @@ public void testNtileNotSupportedWithFrame() @Test public void testInGroupByLimitOutGroupByOrderBy() { - cannotVectorize(); - testBuilder() .sql( "with t AS (SELECT m2, COUNT(m1) as trend_score\n" @@ -15125,7 +14999,6 @@ public void testInGroupByLimitOutGroupByOrderBy() @Test public void testInGroupByOrderByLimitOutGroupByOrderByLimit() { - cannotVectorize(); String sql = "with t AS (SELECT m2 as mo, COUNT(m1) as trend_score\n" + "FROM \"foo\"\n" + "GROUP BY 1\n" @@ -15220,7 +15093,6 @@ public void testScanAndSortCanGetSchemaFromScanQuery() @Test public void testWindowingWithScanAndSort() { - cannotVectorize(); msqIncompatible(); String sql = "with t AS (\n" + "SELECT \n" @@ -15451,7 +15323,6 @@ public void testWindowingOverJoin() @Test public void testCastCharToVarcharInFlattenConcat() { - cannotVectorize(); testQuery( "select 'A'||cast(col as char)||'B' from (values(1)) as t(col)", ImmutableList.of( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java index 2592d2d496cd..d6d53481bbf6 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java @@ -480,7 +480,6 @@ public void testBitwiseExpressions() @Test public void testSafeDivideWithoutTable() { - cannotVectorize(); final Map context = new HashMap<>(QUERY_CONTEXT_DEFAULT); testQuery( @@ -657,10 +656,6 @@ public void testSelectStarWithDimFilter() @Test public void testSelectDistinctWithCascadeExtractionFilter() { - if (NullHandling.sqlCompatible()) { - // cannot vectorize due to expression filter - cannotVectorize(); - } testQuery( "SELECT distinct dim1 FROM druid.foo WHERE substring(substring(dim1, 2), 1, 1) = 'e' OR dim2 = 'a'", ImmutableList.of( @@ -705,9 +700,6 @@ public void testSelectDistinctWithCascadeExtractionFilter() @Test public void testSelectDistinctWithStrlenFilter() { - // Cannot vectorize due to usage of expressions. - cannotVectorize(); - testQuery( "SELECT distinct dim1 FROM druid.foo " + "WHERE CHARACTER_LENGTH(dim1) = 3 OR CAST(CHARACTER_LENGTH(dim1) AS varchar) = 3", @@ -1996,7 +1988,6 @@ public void testAggregateFilterInTheAbsenceOfProjection() @Test public void testCountDistinctNonApproximateEmptySet() { - cannotVectorize(); testQuery( PLANNER_CONFIG_DEFAULT.withOverrides( ImmutableMap.of( @@ -2033,7 +2024,6 @@ public void testCountDistinctNonApproximateEmptySet() @Test public void testCountDistinctNonApproximateBasic() { - cannotVectorize(); testQuery( PLANNER_CONFIG_DEFAULT.withOverrides( ImmutableMap.of( @@ -2069,8 +2059,6 @@ public void testCountDistinctNonApproximateBasic() @Test public void testCountDistinctNonApproximateWithFilter() { - cannotVectorize(); - testQuery( PLANNER_CONFIG_DEFAULT.withOverrides( ImmutableMap.of( @@ -2109,8 +2097,6 @@ public void testCountDistinctNonApproximateWithFilter() @Test public void testCountDistinctNonApproximateWithFilterHaving() { - cannotVectorize(); - testQuery( PLANNER_CONFIG_DEFAULT.withOverrides( ImmutableMap.of( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java index 36dbbe64d80b..3ad1217f84d3 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java @@ -103,9 +103,9 @@ public static Iterable constructorFeeder() @ParameterizedTest(name = "{0}") public void testExactCountDistinctUsingSubqueryWithWhereToOuterFilter(String testName, Map queryContext) { - // Cannot vectorize topN operator. - cannotVectorize(); - + if (!queryContext.containsKey(QueryContexts.MAX_SUBQUERY_BYTES_KEY)) { + cannotVectorize(); + } testQuery( "SELECT\n" + " SUM(cnt),\n" @@ -374,8 +374,9 @@ public void testViewAndJoin(String testName, Map queryContext) @ParameterizedTest(name = "{0}") public void testGroupByWithPostAggregatorReferencingTimeFloorColumnOnTimeseries(String testName, Map queryContext) { - cannotVectorize(); - + if (!queryContext.containsKey(QueryContexts.MAX_SUBQUERY_BYTES_KEY)) { + cannotVectorize(); + } testQuery( "SELECT TIME_FORMAT(\"date\", 'yyyy-MM'), SUM(x)\n" + "FROM (\n" @@ -537,9 +538,9 @@ public void testUsingSubqueryAsFilterOnTwoColumns(String testName, Map queryContext) { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - + if (!queryContext.containsKey(QueryContexts.MAX_SUBQUERY_BYTES_KEY)) { + cannotVectorize(); + } testQuery( "SELECT * FROM (" + " SELECT max(cnt), min(cnt), avg(cnt), TIME_EXTRACT(max(t), 'EPOCH') last_time, count(1) num_days FROM (\n" @@ -782,8 +783,9 @@ public void testZeroMaxNumericInFilter(String testName, Map quer @ParameterizedTest(name = "{0}") public void testUseTimeFloorInsteadOfGranularityOnJoinResult(String testName, Map queryContext) { - cannotVectorize(); - + if (!queryContext.containsKey(QueryContexts.MAX_SUBQUERY_BYTES_KEY)) { + cannotVectorize(); + } testQuery( "WITH main AS (SELECT * FROM foo LIMIT 2)\n" + "SELECT TIME_FLOOR(__time, 'PT1H') AS \"time\", dim1, COUNT(*)\n" @@ -902,9 +904,9 @@ public void testJoinWithTimeDimension(String testName, Map query @ParameterizedTest(name = "{0}") public void testUsingSubqueryWithLimit(String testName, Map queryContext) { - // Cannot vectorize scan query. - cannotVectorize(); - + if (!queryContext.containsKey(QueryContexts.MAX_SUBQUERY_BYTES_KEY)) { + cannotVectorize(); + } testQuery( "SELECT COUNT(*) AS cnt FROM ( SELECT * FROM druid.foo LIMIT 10 ) tmpA", queryContext, @@ -987,8 +989,6 @@ public void testSelfJoin(String testName, Map queryContext) @ParameterizedTest(name = "{0}") public void testJoinWithSubqueries(String testName, Map queryContext) { - cannotVectorize(); - List results = new ArrayList<>(ImmutableList.of( new Object[]{"", NullHandling.defaultStringValue()}, new Object[]{"10.1", NullHandling.defaultStringValue()}, diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteTimeBoundaryQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteTimeBoundaryQueryTest.java index 54ff1a2c00e0..a6be8f7cf99a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteTimeBoundaryQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteTimeBoundaryQueryTest.java @@ -141,9 +141,6 @@ public void testMinTimeQueryWithTimeAndColumnFilters() @Test public void testMinTimeQueryWithTimeAndExpressionFilters() { - // Cannot vectorize due to UPPER expression. - cannotVectorize(); - HashMap queryContext = new HashMap<>(QUERY_CONTEXT_DEFAULT); queryContext.put(QueryContexts.TIME_BOUNDARY_PLANNING_KEY, true); testQuery( @@ -200,9 +197,6 @@ public void testMinMaxTimeQuery() @Test public void testMaxTimeQueryWithJoin() { - // Cannot vectorize due to JOIN. - cannotVectorize(); - HashMap context = new HashMap<>(QUERY_CONTEXT_DEFAULT); context.put(QueryContexts.TIME_BOUNDARY_PLANNING_KEY, true); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/QueryTestRunner.java b/sql/src/test/java/org/apache/druid/sql/calcite/QueryTestRunner.java index df14aeb105a0..4f7fffadfce6 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/QueryTestRunner.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/QueryTestRunner.java @@ -606,12 +606,15 @@ public void verify() // times. Pick the first failure as that emulates the original code flow // where the first exception ended the test. for (QueryResults queryResults : execStep.results()) { - if (queryResults.exception == null) { - continue; - } - // Delayed exception checking to let other verify steps run before running vectorized checks if (builder.queryCannotVectorize && "force".equals(queryResults.vectorizeOption)) { + if (queryResults.exception == null) { + Assert.fail( + "Expected vectorized execution to fail, but it did not. " + + "Please remove cannotVectorize() from this test case." + ); + } + MatcherAssert.assertThat( queryResults.exception, CoreMatchers.allOf( @@ -621,7 +624,7 @@ public void verify() ) ) ); - } else { + } else if (queryResults.exception != null) { throw queryResults.exception; } }