From 8ee414dfb0b42971e85143e21901848cc684ab4a Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Sun, 24 May 2026 20:32:15 -0700 Subject: [PATCH 1/3] feat: add simd add/sub/mul expressions --- benchmarks/pom.xml | 13 +++ .../query/SqlExpressionBenchmark.java | 16 +++ docs/operations/java.md | 6 +- examples/bin/run-java | 1 + .../indexing/overlord/ForkingTaskRunner.java | 3 +- pom.xml | 9 ++ .../druid/math/expr/ExpressionProcessing.java | 22 ++++- .../math/expr/ExpressionProcessingConfig.java | 13 ++- ...leVectorMathBivariateProcessorFactory.java | 55 +++++++++++ .../expr/vector/VectorMathProcessors.java | 14 ++- .../simd/SimdDoubleDoubleAddProcessor.java | 93 ++++++++++++++++++ .../simd/SimdDoubleDoubleMulProcessor.java | 93 ++++++++++++++++++ .../simd/SimdDoubleDoubleProcessor.java | 95 ++++++++++++++++++ .../simd/SimdDoubleDoubleSubProcessor.java | 93 ++++++++++++++++++ .../simd/SimdDoubleLongAddProcessor.java | 96 ++++++++++++++++++ .../simd/SimdDoubleLongMulProcessor.java | 96 ++++++++++++++++++ .../vector/simd/SimdDoubleLongProcessor.java | 98 +++++++++++++++++++ .../simd/SimdDoubleLongSubProcessor.java | 96 ++++++++++++++++++ .../simd/SimdLongDoubleAddProcessor.java | 96 ++++++++++++++++++ .../simd/SimdLongDoubleMulProcessor.java | 96 ++++++++++++++++++ .../vector/simd/SimdLongDoubleProcessor.java | 98 +++++++++++++++++++ .../simd/SimdLongDoubleSubProcessor.java | 96 ++++++++++++++++++ .../vector/simd/SimdLongLongAddProcessor.java | 93 ++++++++++++++++++ .../vector/simd/SimdLongLongMulProcessor.java | 93 ++++++++++++++++++ .../vector/simd/SimdLongLongProcessor.java | 96 ++++++++++++++++++ .../vector/simd/SimdLongLongSubProcessor.java | 93 ++++++++++++++++++ .../math/expr/vector/simd/SimdProcessors.java | 98 +++++++++++++++++++ .../vector/simd/SimdSupportedBinaryOp.java | 36 +++++++ .../expr/VectorExprResultConsistencyTest.java | 42 +++++--- ...torExprResultConsistencyVectorApiTest.java | 44 +++++++++ 30 files changed, 1869 insertions(+), 24 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleDoubleAddProcessor.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleDoubleMulProcessor.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleDoubleProcessor.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleDoubleSubProcessor.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleLongAddProcessor.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleLongMulProcessor.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleLongProcessor.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleLongSubProcessor.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongDoubleAddProcessor.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongDoubleMulProcessor.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongDoubleProcessor.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongDoubleSubProcessor.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongLongAddProcessor.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongLongMulProcessor.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongLongProcessor.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongLongSubProcessor.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdProcessors.java create mode 100644 processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdSupportedBinaryOp.java create mode 100644 processing/src/test/java/org/apache/druid/math/expr/VectorExprResultConsistencyVectorApiTest.java diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 429b9323777a..12fec8bb4e72 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -244,6 +244,19 @@ + + + org.apache.maven.plugins + maven-compiler-plugin + 3.14.1 + true + + ${maven.compiler.release} + + org.openjdk.jmh.generators.BenchmarkProcessor + + + org.apache.maven.plugins maven-assembly-plugin diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java index 8988973f9825..361eddff4949 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java @@ -21,12 +21,15 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.groupby.GroupByQueryConfig; import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.Warmup; @@ -171,6 +174,19 @@ public class SqlExpressionBenchmark extends SqlBaseQueryBenchmark }) private String deferExpressionDimensions; + @Param({"false", "true"}) + private boolean useVectorApi; + + @Setup(Level.Trial) + public void setupExpressionProcessing() + { + if (useVectorApi) { + ExpressionProcessing.initializeForVectorApiTests(); + } else { + ExpressionProcessing.initializeForTests(); + } + } + @Param({ // non-expression reference "0", diff --git a/docs/operations/java.md b/docs/operations/java.md index f4a8c029db24..c6117e1f4263 100644 --- a/docs/operations/java.md +++ b/docs/operations/java.md @@ -85,5 +85,9 @@ added. There are many ways of doing this. Choose the one that works best for you --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \ --add-opens=java.base/java.io=ALL-UNNAMED \ --add-opens=java.base/java.lang=ALL-UNNAMED \ ---add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED +--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED \ +--add-modules=jdk.incubator.vector ``` + +The `--add-modules=jdk.incubator.vector` flag is optional, but adding it makes the JDK's incubator Vector API available +to Druid to support `druid.expressions.useVectorApi=true`. diff --git a/examples/bin/run-java b/examples/bin/run-java index 80190d0a793c..5a30cd54fbdf 100755 --- a/examples/bin/run-java +++ b/examples/bin/run-java @@ -43,6 +43,7 @@ then --add-opens=java.base/java.io=ALL-UNNAMED \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED \ + --add-modules=jdk.incubator.vector \ "$@" else exec "$JAVA_BIN" "$@" diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/ForkingTaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/ForkingTaskRunner.java index 1f7e42181071..0cb7f8a51e1d 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/ForkingTaskRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/ForkingTaskRunner.java @@ -115,7 +115,8 @@ public class ForkingTaskRunner "--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED", "--add-opens=java.base/java.io=ALL-UNNAMED", "--add-opens=java.base/java.lang=ALL-UNNAMED", - "--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED" + "--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED", + "--add-modules=jdk.incubator.vector" ); private final ForkingTaskRunnerConfig config; diff --git a/pom.xml b/pom.xml index 7c3752b31814..1f0640b03a24 100644 --- a/pom.xml +++ b/pom.xml @@ -163,6 +163,9 @@ --add-opens=java.base/java.util=ALL-UNNAMED + + + --add-modules=jdk.incubator.vector maven.org @@ -1793,6 +1796,8 @@ **/*_jmhType_*.class **/*_jmhTest_*.class **/*_generated*.class + + **/math/expr/vector/simd/Simd*.class **.SuppressForbidden @@ -2151,6 +2156,9 @@ true ${maven.compiler.release} + + --add-modules=jdk.incubator.vector + @@ -2211,6 +2219,7 @@ -J--add-exports=java.base/sun.nio.ch=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED + -J--add-modules=jdk.incubator.vector diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java index 7b2e1d26ae49..68b5cef8cc19 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java +++ b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessing.java @@ -45,13 +45,19 @@ public class ExpressionProcessing @VisibleForTesting public static void initializeForTests() { - INSTANCE = new ExpressionProcessingConfig(null, null, null); + INSTANCE = new ExpressionProcessingConfig(null, null, null, null); } @VisibleForTesting public static void initializeForHomogenizeNullMultiValueStrings() { - INSTANCE = new ExpressionProcessingConfig(null, true, null); + INSTANCE = new ExpressionProcessingConfig(null, true, null, null); + } + + @VisibleForTesting + public static void initializeForVectorApiTests() + { + INSTANCE = new ExpressionProcessingConfig(null, null, null, true); } /** @@ -81,6 +87,18 @@ public static boolean allowVectorizeFallback() return INSTANCE.allowVectorizeFallback(); } + /** + * Whether {@link org.apache.druid.math.expr.vector.ExprVectorProcessor} implementations may dispatch to specialized + * {@code jdk.incubator.vector} (SIMD) variants for supported math operations. Off by default; opt-in via + * {@link ExpressionProcessingConfig#USE_VECTOR_API}. Requires the JVM to be started with + * {@code --add-modules=jdk.incubator.vector}, which Druid already adds to its standard launch arguments. + */ + public static boolean useVectorApi() + { + checkInitialized(); + return INSTANCE.useVectorApi(); + } + private static void checkInitialized() { // this should only be null in a unit test context, in production this will be injected by the null handling module diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java index 78c2ecfcbbc9..d24600a302a7 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java +++ b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java @@ -34,6 +34,7 @@ public class ExpressionProcessingConfig public static final String HOMOGENIZE_NULL_MULTIVALUE_STRING_ARRAYS = "druid.expressions.homogenizeNullMultiValueStringArrays"; public static final String ALLOW_VECTORIZE_FALLBACK = "druid.expressions.allowVectorizeFallback"; + public static final String USE_VECTOR_API = "druid.expressions.useVectorApi"; @JsonProperty("processArraysAsMultiValueStrings") private final boolean processArraysAsMultiValueStrings; @@ -44,11 +45,15 @@ public class ExpressionProcessingConfig @JsonProperty("allowVectorizeFallback") private final boolean allowVectorizeFallback; + @JsonProperty("useVectorApi") + private final boolean useVectorApi; + @JsonCreator public ExpressionProcessingConfig( @JsonProperty("processArraysAsMultiValueStrings") @Nullable Boolean processArraysAsMultiValueStrings, @JsonProperty("homogenizeNullMultiValueStringArrays") @Nullable Boolean homogenizeNullMultiValueStringArrays, - @JsonProperty("allowVectorizeFallback") @Nullable Boolean allowVectorizeFallback + @JsonProperty("allowVectorizeFallback") @Nullable Boolean allowVectorizeFallback, + @JsonProperty("useVectorApi") @Nullable Boolean useVectorApi ) { this.processArraysAsMultiValueStrings = getWithPropertyFallbackFalse( @@ -64,6 +69,7 @@ public ExpressionProcessingConfig( ALLOW_VECTORIZE_FALLBACK, "true" ); + this.useVectorApi = getWithPropertyFallbackFalse(useVectorApi, USE_VECTOR_API); } public boolean processArraysAsMultiValueStrings() @@ -81,6 +87,11 @@ public boolean allowVectorizeFallback() return allowVectorizeFallback; } + public boolean useVectorApi() + { + return useVectorApi; + } + private static boolean getWithPropertyFallbackFalse(@Nullable Boolean value, String property) { return getWithPropertyFallback(value, property, "false"); diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/SimpleVectorMathBivariateProcessorFactory.java b/processing/src/main/java/org/apache/druid/math/expr/vector/SimpleVectorMathBivariateProcessorFactory.java index 230b2e44f350..deebbeb565ff 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/SimpleVectorMathBivariateProcessorFactory.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/SimpleVectorMathBivariateProcessorFactory.java @@ -20,10 +20,15 @@ package org.apache.druid.math.expr.vector; import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.math.expr.vector.functional.DoubleBivariateDoubleLongFunction; import org.apache.druid.math.expr.vector.functional.DoubleBivariateDoublesFunction; import org.apache.druid.math.expr.vector.functional.DoubleBivariateLongDoubleFunction; import org.apache.druid.math.expr.vector.functional.LongBivariateLongsFunction; +import org.apache.druid.math.expr.vector.simd.SimdProcessors; +import org.apache.druid.math.expr.vector.simd.SimdSupportedBinaryOp; + +import javax.annotation.Nullable; /** * Make a 2 argument, math processor with the following type rules @@ -31,6 +36,10 @@ * long, double -> double * double, long -> double * double, double -> double + * + * If a non-null {@link SimdSupportedBinaryOp} is supplied to the constructor and + * {@link ExpressionProcessing#useVectorApi()} is true, this factory will return SIMD-specialized processors backed + * by the JDK incubator {@code jdk.incubator.vector} API instead of the standard scalar implementations. */ public class SimpleVectorMathBivariateProcessorFactory extends VectorMathBivariateProcessorFactory { @@ -38,6 +47,8 @@ public class SimpleVectorMathBivariateProcessorFactory extends VectorMathBivaria private final DoubleBivariateLongDoubleFunction longDoubleFunction; private final DoubleBivariateDoubleLongFunction doubleLongFunction; private final DoubleBivariateDoublesFunction doublesFunction; + @Nullable + private final SimdSupportedBinaryOp simdOp; protected SimpleVectorMathBivariateProcessorFactory( LongBivariateLongsFunction longsFunction, @@ -45,16 +56,36 @@ protected SimpleVectorMathBivariateProcessorFactory( DoubleBivariateDoubleLongFunction doubleLongFunction, DoubleBivariateDoublesFunction doublesFunction ) + { + this(longsFunction, longDoubleFunction, doubleLongFunction, doublesFunction, null); + } + + protected SimpleVectorMathBivariateProcessorFactory( + LongBivariateLongsFunction longsFunction, + DoubleBivariateLongDoubleFunction longDoubleFunction, + DoubleBivariateDoubleLongFunction doubleLongFunction, + DoubleBivariateDoublesFunction doublesFunction, + @Nullable SimdSupportedBinaryOp simdOp + ) { this.longsFunction = longsFunction; this.longDoubleFunction = longDoubleFunction; this.doubleLongFunction = doubleLongFunction; this.doublesFunction = doublesFunction; + this.simdOp = simdOp; } @Override public final ExprVectorProcessor longsProcessor(Expr.VectorInputBindingInspector inspector, Expr left, Expr right) { + if (simdOp != null && ExpressionProcessing.useVectorApi()) { + return SimdProcessors.makeLongLong( + left.asVectorProcessor(inspector), + right.asVectorProcessor(inspector), + simdOp, + longsFunction + ); + } return new LongBivariateLongsFunctionVectorProcessor( left.asVectorProcessor(inspector), right.asVectorProcessor(inspector), @@ -69,6 +100,14 @@ public final ExprVectorProcessor longDoubleProcessor( Expr right ) { + if (simdOp != null && ExpressionProcessing.useVectorApi()) { + return SimdProcessors.makeLongDouble( + left.asVectorProcessor(inspector), + right.asVectorProcessor(inspector), + simdOp, + longDoubleFunction + ); + } return new DoubleBivariateLongDoubleFunctionVectorProcessor( left.asVectorProcessor(inspector), right.asVectorProcessor(inspector), @@ -83,6 +122,14 @@ public final ExprVectorProcessor doubleLongProcessor( Expr right ) { + if (simdOp != null && ExpressionProcessing.useVectorApi()) { + return SimdProcessors.makeDoubleLong( + left.asVectorProcessor(inspector), + right.asVectorProcessor(inspector), + simdOp, + doubleLongFunction + ); + } return new DoubleBivariateDoubleLongFunctionVectorProcessor( left.asVectorProcessor(inspector), right.asVectorProcessor(inspector), @@ -97,6 +144,14 @@ public final ExprVectorProcessor doublesProcessor( Expr right ) { + if (simdOp != null && ExpressionProcessing.useVectorApi()) { + return SimdProcessors.makeDoubleDouble( + left.asVectorProcessor(inspector), + right.asVectorProcessor(inspector), + simdOp, + doublesFunction + ); + } return new DoubleBivariateDoublesFunctionVectorProcessor( left.asVectorProcessor(inspector), right.asVectorProcessor(inspector), diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/VectorMathProcessors.java b/processing/src/main/java/org/apache/druid/math/expr/vector/VectorMathProcessors.java index c12ebc55eaa9..4a26f8141531 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/VectorMathProcessors.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/VectorMathProcessors.java @@ -23,6 +23,7 @@ import com.google.common.primitives.Ints; import org.apache.druid.math.expr.ExpressionValidationException; import org.apache.druid.math.expr.Function; +import org.apache.druid.math.expr.vector.simd.SimdSupportedBinaryOp; public class VectorMathProcessors { @@ -300,7 +301,7 @@ public static final class Add extends SimpleVectorMathBivariateProcessorFactory public Add() { - super(Long::sum, Double::sum, Double::sum, Double::sum); + super(Long::sum, Double::sum, Double::sum, Double::sum, SimdSupportedBinaryOp.ADD); } } @@ -314,7 +315,8 @@ public Subtract() (left, right) -> left - right, (left, right) -> (double) left - right, (left, right) -> left - (double) right, - (left, right) -> left - right + (left, right) -> left - right, + SimdSupportedBinaryOp.SUB ); } } @@ -325,7 +327,13 @@ public static final class Multiply extends SimpleVectorMathBivariateProcessorFac public Multiply() { - super(Multiply::multiply, Multiply::multiply, Multiply::multiply, Multiply::multiply); + super( + Multiply::multiply, + Multiply::multiply, + Multiply::multiply, + Multiply::multiply, + SimdSupportedBinaryOp.MUL + ); } private static long multiply(long x, long y) diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleDoubleAddProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleDoubleAddProcessor.java new file mode 100644 index 000000000000..d476468cb076 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleDoubleAddProcessor.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import jdk.incubator.vector.DoubleVector; +import jdk.incubator.vector.VectorMask; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.DoubleBivariateDoublesFunction; + +import java.util.Arrays; + +/** + * SIMD specialization of {@code (double[], double[]) -> double[]} addition. The op is hardcoded to + * {@link DoubleVector#add} so the JIT statically resolves it to the platform's double-add intrinsic. + */ +public final class SimdDoubleDoubleAddProcessor extends SimdDoubleDoubleProcessor +{ + public SimdDoubleDoubleAddProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + DoubleBivariateDoublesFunction scalarFallback + ) + { + super(left, right, scalarFallback); + } + + @Override + protected void processVector( + double[] leftInput, + double[] rightInput, + boolean[] leftNulls, + boolean[] rightNulls, + int currentSize + ) + { + final boolean hasLeftNulls = leftNulls != null; + final boolean hasRightNulls = rightNulls != null; + final int laneCount = SPECIES.length(); + final int upperBound = SPECIES.loopBound(currentSize); + int i = 0; + if (!hasLeftNulls && !hasRightNulls) { + for (; i < upperBound; i += laneCount) { + final DoubleVector va = DoubleVector.fromArray(SPECIES, leftInput, i); + final DoubleVector vb = DoubleVector.fromArray(SPECIES, rightInput, i); + va.add(vb).intoArray(outValues, i); + } + for (; i < currentSize; i++) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + Arrays.fill(outNulls, 0, currentSize, false); + } else { + for (; i < upperBound; i += laneCount) { + final VectorMask nm; + if (hasLeftNulls && hasRightNulls) { + nm = VectorMask.fromArray(SPECIES, leftNulls, i) + .or(VectorMask.fromArray(SPECIES, rightNulls, i)); + } else if (hasLeftNulls) { + nm = VectorMask.fromArray(SPECIES, leftNulls, i); + } else { + nm = VectorMask.fromArray(SPECIES, rightNulls, i); + } + final DoubleVector va = DoubleVector.fromArray(SPECIES, leftInput, i); + final DoubleVector vb = DoubleVector.fromArray(SPECIES, rightInput, i); + va.add(vb).intoArray(outValues, i); + nm.intoArray(outNulls, i); + } + for (; i < currentSize; i++) { + final boolean isNull = (hasLeftNulls && leftNulls[i]) || (hasRightNulls && rightNulls[i]); + outNulls[i] = isNull; + if (!isNull) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + } + } + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleDoubleMulProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleDoubleMulProcessor.java new file mode 100644 index 000000000000..56cf53e3309e --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleDoubleMulProcessor.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import jdk.incubator.vector.DoubleVector; +import jdk.incubator.vector.VectorMask; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.DoubleBivariateDoublesFunction; + +import java.util.Arrays; + +/** + * SIMD specialization of {@code (double[], double[]) -> double[]} multiplication. The op is hardcoded to + * {@link DoubleVector#mul} so the JIT statically resolves it to the platform's double-multiply intrinsic. + */ +public final class SimdDoubleDoubleMulProcessor extends SimdDoubleDoubleProcessor +{ + public SimdDoubleDoubleMulProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + DoubleBivariateDoublesFunction scalarFallback + ) + { + super(left, right, scalarFallback); + } + + @Override + protected void processVector( + double[] leftInput, + double[] rightInput, + boolean[] leftNulls, + boolean[] rightNulls, + int currentSize + ) + { + final boolean hasLeftNulls = leftNulls != null; + final boolean hasRightNulls = rightNulls != null; + final int laneCount = SPECIES.length(); + final int upperBound = SPECIES.loopBound(currentSize); + int i = 0; + if (!hasLeftNulls && !hasRightNulls) { + for (; i < upperBound; i += laneCount) { + final DoubleVector va = DoubleVector.fromArray(SPECIES, leftInput, i); + final DoubleVector vb = DoubleVector.fromArray(SPECIES, rightInput, i); + va.mul(vb).intoArray(outValues, i); + } + for (; i < currentSize; i++) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + Arrays.fill(outNulls, 0, currentSize, false); + } else { + for (; i < upperBound; i += laneCount) { + final VectorMask nm; + if (hasLeftNulls && hasRightNulls) { + nm = VectorMask.fromArray(SPECIES, leftNulls, i) + .or(VectorMask.fromArray(SPECIES, rightNulls, i)); + } else if (hasLeftNulls) { + nm = VectorMask.fromArray(SPECIES, leftNulls, i); + } else { + nm = VectorMask.fromArray(SPECIES, rightNulls, i); + } + final DoubleVector va = DoubleVector.fromArray(SPECIES, leftInput, i); + final DoubleVector vb = DoubleVector.fromArray(SPECIES, rightInput, i); + va.mul(vb).intoArray(outValues, i); + nm.intoArray(outNulls, i); + } + for (; i < currentSize; i++) { + final boolean isNull = (hasLeftNulls && leftNulls[i]) || (hasRightNulls && rightNulls[i]); + outNulls[i] = isNull; + if (!isNull) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + } + } + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleDoubleProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleDoubleProcessor.java new file mode 100644 index 000000000000..8f3eeebac2c8 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleDoubleProcessor.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import jdk.incubator.vector.DoubleVector; +import jdk.incubator.vector.VectorSpecies; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExpressionType; +import org.apache.druid.math.expr.vector.CastToTypeVectorProcessor; +import org.apache.druid.math.expr.vector.ExprEvalDoubleVector; +import org.apache.druid.math.expr.vector.ExprEvalVector; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.DoubleBivariateDoublesFunction; + +import javax.annotation.Nullable; + +/** + * Abstract base for SIMD processors that compute {@code (double[], double[]) -> double[]} ops. See + * {@link SimdLongLongProcessor} for the design rationale. + */ +abstract class SimdDoubleDoubleProcessor implements ExprVectorProcessor +{ + static final VectorSpecies SPECIES = DoubleVector.SPECIES_PREFERRED; + + private final ExprVectorProcessor left; + private final ExprVectorProcessor right; + final DoubleBivariateDoublesFunction scalarFallback; + final double[] outValues; + final boolean[] outNulls; + + protected SimdDoubleDoubleProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + DoubleBivariateDoublesFunction scalarFallback + ) + { + this.left = CastToTypeVectorProcessor.cast(left, ExpressionType.DOUBLE); + this.right = CastToTypeVectorProcessor.cast(right, ExpressionType.DOUBLE); + this.scalarFallback = scalarFallback; + this.outValues = new double[this.left.maxVectorSize()]; + this.outNulls = new boolean[this.left.maxVectorSize()]; + } + + @Override + public final ExprEvalVector evalVector(Expr.VectorInputBinding bindings) + { + final ExprEvalVector lhs = left.evalVector(bindings); + final ExprEvalVector rhs = right.evalVector(bindings); + processVector( + lhs.values(), + rhs.values(), + lhs.getNullVector(), + rhs.getNullVector(), + bindings.getCurrentVectorSize() + ); + return new ExprEvalDoubleVector(outValues, outNulls); + } + + protected abstract void processVector( + double[] leftInput, + double[] rightInput, + @Nullable boolean[] leftNulls, + @Nullable boolean[] rightNulls, + int currentSize + ); + + @Override + public final ExpressionType getOutputType() + { + return ExpressionType.DOUBLE; + } + + @Override + public final int maxVectorSize() + { + return outValues.length; + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleDoubleSubProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleDoubleSubProcessor.java new file mode 100644 index 000000000000..9f290240bce8 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleDoubleSubProcessor.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import jdk.incubator.vector.DoubleVector; +import jdk.incubator.vector.VectorMask; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.DoubleBivariateDoublesFunction; + +import java.util.Arrays; + +/** + * SIMD specialization of {@code (double[], double[]) -> double[]} subtraction. The op is hardcoded to + * {@link DoubleVector#sub} so the JIT statically resolves it to the platform's double-subtract intrinsic. + */ +public final class SimdDoubleDoubleSubProcessor extends SimdDoubleDoubleProcessor +{ + public SimdDoubleDoubleSubProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + DoubleBivariateDoublesFunction scalarFallback + ) + { + super(left, right, scalarFallback); + } + + @Override + protected void processVector( + double[] leftInput, + double[] rightInput, + boolean[] leftNulls, + boolean[] rightNulls, + int currentSize + ) + { + final boolean hasLeftNulls = leftNulls != null; + final boolean hasRightNulls = rightNulls != null; + final int laneCount = SPECIES.length(); + final int upperBound = SPECIES.loopBound(currentSize); + int i = 0; + if (!hasLeftNulls && !hasRightNulls) { + for (; i < upperBound; i += laneCount) { + final DoubleVector va = DoubleVector.fromArray(SPECIES, leftInput, i); + final DoubleVector vb = DoubleVector.fromArray(SPECIES, rightInput, i); + va.sub(vb).intoArray(outValues, i); + } + for (; i < currentSize; i++) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + Arrays.fill(outNulls, 0, currentSize, false); + } else { + for (; i < upperBound; i += laneCount) { + final VectorMask nm; + if (hasLeftNulls && hasRightNulls) { + nm = VectorMask.fromArray(SPECIES, leftNulls, i) + .or(VectorMask.fromArray(SPECIES, rightNulls, i)); + } else if (hasLeftNulls) { + nm = VectorMask.fromArray(SPECIES, leftNulls, i); + } else { + nm = VectorMask.fromArray(SPECIES, rightNulls, i); + } + final DoubleVector va = DoubleVector.fromArray(SPECIES, leftInput, i); + final DoubleVector vb = DoubleVector.fromArray(SPECIES, rightInput, i); + va.sub(vb).intoArray(outValues, i); + nm.intoArray(outNulls, i); + } + for (; i < currentSize; i++) { + final boolean isNull = (hasLeftNulls && leftNulls[i]) || (hasRightNulls && rightNulls[i]); + outNulls[i] = isNull; + if (!isNull) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + } + } + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleLongAddProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleLongAddProcessor.java new file mode 100644 index 000000000000..5d1eb74f0d96 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleLongAddProcessor.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import jdk.incubator.vector.DoubleVector; +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorMask; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.DoubleBivariateDoubleLongFunction; + +import java.util.Arrays; + +/** + * SIMD specialization of {@code (double[], long[]) -> double[]} addition. The op is hardcoded to + * {@link DoubleVector#add} so the JIT statically resolves it to the platform's double-add intrinsic. + */ +public final class SimdDoubleLongAddProcessor extends SimdDoubleLongProcessor +{ + public SimdDoubleLongAddProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + DoubleBivariateDoubleLongFunction scalarFallback + ) + { + super(left, right, scalarFallback); + } + + @Override + protected void processVector( + double[] leftInput, + long[] rightInput, + boolean[] leftNulls, + boolean[] rightNulls, + int currentSize + ) + { + final boolean hasLeftNulls = leftNulls != null; + final boolean hasRightNulls = rightNulls != null; + final int laneCount = DOUBLE_SPECIES.length(); + final int upperBound = DOUBLE_SPECIES.loopBound(currentSize); + int i = 0; + if (!hasLeftNulls && !hasRightNulls) { + for (; i < upperBound; i += laneCount) { + final DoubleVector va = DoubleVector.fromArray(DOUBLE_SPECIES, leftInput, i); + final DoubleVector vb = + (DoubleVector) LongVector.fromArray(LONG_SPECIES, rightInput, i).castShape(DOUBLE_SPECIES, 0); + va.add(vb).intoArray(outValues, i); + } + for (; i < currentSize; i++) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + Arrays.fill(outNulls, 0, currentSize, false); + } else { + for (; i < upperBound; i += laneCount) { + final VectorMask nm; + if (hasLeftNulls && hasRightNulls) { + nm = VectorMask.fromArray(DOUBLE_SPECIES, leftNulls, i) + .or(VectorMask.fromArray(DOUBLE_SPECIES, rightNulls, i)); + } else if (hasLeftNulls) { + nm = VectorMask.fromArray(DOUBLE_SPECIES, leftNulls, i); + } else { + nm = VectorMask.fromArray(DOUBLE_SPECIES, rightNulls, i); + } + final DoubleVector va = DoubleVector.fromArray(DOUBLE_SPECIES, leftInput, i); + final DoubleVector vb = + (DoubleVector) LongVector.fromArray(LONG_SPECIES, rightInput, i).castShape(DOUBLE_SPECIES, 0); + va.add(vb).intoArray(outValues, i); + nm.intoArray(outNulls, i); + } + for (; i < currentSize; i++) { + final boolean isNull = (hasLeftNulls && leftNulls[i]) || (hasRightNulls && rightNulls[i]); + outNulls[i] = isNull; + if (!isNull) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + } + } + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleLongMulProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleLongMulProcessor.java new file mode 100644 index 000000000000..b593799ee262 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleLongMulProcessor.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import jdk.incubator.vector.DoubleVector; +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorMask; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.DoubleBivariateDoubleLongFunction; + +import java.util.Arrays; + +/** + * SIMD specialization of {@code (double[], long[]) -> double[]} multiplication. The op is hardcoded to + * {@link DoubleVector#mul} so the JIT statically resolves it to the platform's double-multiply intrinsic. + */ +public final class SimdDoubleLongMulProcessor extends SimdDoubleLongProcessor +{ + public SimdDoubleLongMulProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + DoubleBivariateDoubleLongFunction scalarFallback + ) + { + super(left, right, scalarFallback); + } + + @Override + protected void processVector( + double[] leftInput, + long[] rightInput, + boolean[] leftNulls, + boolean[] rightNulls, + int currentSize + ) + { + final boolean hasLeftNulls = leftNulls != null; + final boolean hasRightNulls = rightNulls != null; + final int laneCount = DOUBLE_SPECIES.length(); + final int upperBound = DOUBLE_SPECIES.loopBound(currentSize); + int i = 0; + if (!hasLeftNulls && !hasRightNulls) { + for (; i < upperBound; i += laneCount) { + final DoubleVector va = DoubleVector.fromArray(DOUBLE_SPECIES, leftInput, i); + final DoubleVector vb = + (DoubleVector) LongVector.fromArray(LONG_SPECIES, rightInput, i).castShape(DOUBLE_SPECIES, 0); + va.mul(vb).intoArray(outValues, i); + } + for (; i < currentSize; i++) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + Arrays.fill(outNulls, 0, currentSize, false); + } else { + for (; i < upperBound; i += laneCount) { + final VectorMask nm; + if (hasLeftNulls && hasRightNulls) { + nm = VectorMask.fromArray(DOUBLE_SPECIES, leftNulls, i) + .or(VectorMask.fromArray(DOUBLE_SPECIES, rightNulls, i)); + } else if (hasLeftNulls) { + nm = VectorMask.fromArray(DOUBLE_SPECIES, leftNulls, i); + } else { + nm = VectorMask.fromArray(DOUBLE_SPECIES, rightNulls, i); + } + final DoubleVector va = DoubleVector.fromArray(DOUBLE_SPECIES, leftInput, i); + final DoubleVector vb = + (DoubleVector) LongVector.fromArray(LONG_SPECIES, rightInput, i).castShape(DOUBLE_SPECIES, 0); + va.mul(vb).intoArray(outValues, i); + nm.intoArray(outNulls, i); + } + for (; i < currentSize; i++) { + final boolean isNull = (hasLeftNulls && leftNulls[i]) || (hasRightNulls && rightNulls[i]); + outNulls[i] = isNull; + if (!isNull) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + } + } + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleLongProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleLongProcessor.java new file mode 100644 index 000000000000..e3e705d656a1 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleLongProcessor.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import jdk.incubator.vector.DoubleVector; +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorSpecies; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExpressionType; +import org.apache.druid.math.expr.vector.CastToTypeVectorProcessor; +import org.apache.druid.math.expr.vector.ExprEvalDoubleVector; +import org.apache.druid.math.expr.vector.ExprEvalVector; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.DoubleBivariateDoubleLongFunction; + +import javax.annotation.Nullable; + +/** + * Abstract base for SIMD processors that compute {@code (double[], long[]) -> double[]} ops. The long lane is + * widened to {@link DoubleVector} via {@code castShape(DoubleVector.SPECIES_PREFERRED, 0)} in each subclass's hot + * loop. See {@link SimdLongLongProcessor} for the design rationale. + */ +abstract class SimdDoubleLongProcessor implements ExprVectorProcessor +{ + static final VectorSpecies LONG_SPECIES = LongVector.SPECIES_PREFERRED; + static final VectorSpecies DOUBLE_SPECIES = DoubleVector.SPECIES_PREFERRED; + + private final ExprVectorProcessor left; + private final ExprVectorProcessor right; + final DoubleBivariateDoubleLongFunction scalarFallback; + final double[] outValues; + final boolean[] outNulls; + + protected SimdDoubleLongProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + DoubleBivariateDoubleLongFunction scalarFallback + ) + { + this.left = CastToTypeVectorProcessor.cast(left, ExpressionType.DOUBLE); + this.right = CastToTypeVectorProcessor.cast(right, ExpressionType.LONG); + this.scalarFallback = scalarFallback; + this.outValues = new double[this.left.maxVectorSize()]; + this.outNulls = new boolean[this.left.maxVectorSize()]; + } + + @Override + public final ExprEvalVector evalVector(Expr.VectorInputBinding bindings) + { + final ExprEvalVector lhs = left.evalVector(bindings); + final ExprEvalVector rhs = right.evalVector(bindings); + processVector( + lhs.values(), + rhs.values(), + lhs.getNullVector(), + rhs.getNullVector(), + bindings.getCurrentVectorSize() + ); + return new ExprEvalDoubleVector(outValues, outNulls); + } + + protected abstract void processVector( + double[] leftInput, + long[] rightInput, + @Nullable boolean[] leftNulls, + @Nullable boolean[] rightNulls, + int currentSize + ); + + @Override + public final ExpressionType getOutputType() + { + return ExpressionType.DOUBLE; + } + + @Override + public final int maxVectorSize() + { + return outValues.length; + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleLongSubProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleLongSubProcessor.java new file mode 100644 index 000000000000..97da5a718f60 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdDoubleLongSubProcessor.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import jdk.incubator.vector.DoubleVector; +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorMask; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.DoubleBivariateDoubleLongFunction; + +import java.util.Arrays; + +/** + * SIMD specialization of {@code (double[], long[]) -> double[]} subtraction. The op is hardcoded to + * {@link DoubleVector#sub} so the JIT statically resolves it to the platform's double-subtract intrinsic. + */ +public final class SimdDoubleLongSubProcessor extends SimdDoubleLongProcessor +{ + public SimdDoubleLongSubProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + DoubleBivariateDoubleLongFunction scalarFallback + ) + { + super(left, right, scalarFallback); + } + + @Override + protected void processVector( + double[] leftInput, + long[] rightInput, + boolean[] leftNulls, + boolean[] rightNulls, + int currentSize + ) + { + final boolean hasLeftNulls = leftNulls != null; + final boolean hasRightNulls = rightNulls != null; + final int laneCount = DOUBLE_SPECIES.length(); + final int upperBound = DOUBLE_SPECIES.loopBound(currentSize); + int i = 0; + if (!hasLeftNulls && !hasRightNulls) { + for (; i < upperBound; i += laneCount) { + final DoubleVector va = DoubleVector.fromArray(DOUBLE_SPECIES, leftInput, i); + final DoubleVector vb = + (DoubleVector) LongVector.fromArray(LONG_SPECIES, rightInput, i).castShape(DOUBLE_SPECIES, 0); + va.sub(vb).intoArray(outValues, i); + } + for (; i < currentSize; i++) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + Arrays.fill(outNulls, 0, currentSize, false); + } else { + for (; i < upperBound; i += laneCount) { + final VectorMask nm; + if (hasLeftNulls && hasRightNulls) { + nm = VectorMask.fromArray(DOUBLE_SPECIES, leftNulls, i) + .or(VectorMask.fromArray(DOUBLE_SPECIES, rightNulls, i)); + } else if (hasLeftNulls) { + nm = VectorMask.fromArray(DOUBLE_SPECIES, leftNulls, i); + } else { + nm = VectorMask.fromArray(DOUBLE_SPECIES, rightNulls, i); + } + final DoubleVector va = DoubleVector.fromArray(DOUBLE_SPECIES, leftInput, i); + final DoubleVector vb = + (DoubleVector) LongVector.fromArray(LONG_SPECIES, rightInput, i).castShape(DOUBLE_SPECIES, 0); + va.sub(vb).intoArray(outValues, i); + nm.intoArray(outNulls, i); + } + for (; i < currentSize; i++) { + final boolean isNull = (hasLeftNulls && leftNulls[i]) || (hasRightNulls && rightNulls[i]); + outNulls[i] = isNull; + if (!isNull) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + } + } + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongDoubleAddProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongDoubleAddProcessor.java new file mode 100644 index 000000000000..bd077a05a023 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongDoubleAddProcessor.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import jdk.incubator.vector.DoubleVector; +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorMask; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.DoubleBivariateLongDoubleFunction; + +import java.util.Arrays; + +/** + * SIMD specialization of {@code (long[], double[]) -> double[]} addition. The op is hardcoded to + * {@link DoubleVector#add} so the JIT statically resolves it to the platform's double-add intrinsic. + */ +public final class SimdLongDoubleAddProcessor extends SimdLongDoubleProcessor +{ + public SimdLongDoubleAddProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + DoubleBivariateLongDoubleFunction scalarFallback + ) + { + super(left, right, scalarFallback); + } + + @Override + protected void processVector( + long[] leftInput, + double[] rightInput, + boolean[] leftNulls, + boolean[] rightNulls, + int currentSize + ) + { + final boolean hasLeftNulls = leftNulls != null; + final boolean hasRightNulls = rightNulls != null; + final int laneCount = DOUBLE_SPECIES.length(); + final int upperBound = DOUBLE_SPECIES.loopBound(currentSize); + int i = 0; + if (!hasLeftNulls && !hasRightNulls) { + for (; i < upperBound; i += laneCount) { + final DoubleVector va = + (DoubleVector) LongVector.fromArray(LONG_SPECIES, leftInput, i).castShape(DOUBLE_SPECIES, 0); + final DoubleVector vb = DoubleVector.fromArray(DOUBLE_SPECIES, rightInput, i); + va.add(vb).intoArray(outValues, i); + } + for (; i < currentSize; i++) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + Arrays.fill(outNulls, 0, currentSize, false); + } else { + for (; i < upperBound; i += laneCount) { + final VectorMask nm; + if (hasLeftNulls && hasRightNulls) { + nm = VectorMask.fromArray(DOUBLE_SPECIES, leftNulls, i) + .or(VectorMask.fromArray(DOUBLE_SPECIES, rightNulls, i)); + } else if (hasLeftNulls) { + nm = VectorMask.fromArray(DOUBLE_SPECIES, leftNulls, i); + } else { + nm = VectorMask.fromArray(DOUBLE_SPECIES, rightNulls, i); + } + final DoubleVector va = + (DoubleVector) LongVector.fromArray(LONG_SPECIES, leftInput, i).castShape(DOUBLE_SPECIES, 0); + final DoubleVector vb = DoubleVector.fromArray(DOUBLE_SPECIES, rightInput, i); + va.add(vb).intoArray(outValues, i); + nm.intoArray(outNulls, i); + } + for (; i < currentSize; i++) { + final boolean isNull = (hasLeftNulls && leftNulls[i]) || (hasRightNulls && rightNulls[i]); + outNulls[i] = isNull; + if (!isNull) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + } + } + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongDoubleMulProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongDoubleMulProcessor.java new file mode 100644 index 000000000000..2d211e26b7e1 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongDoubleMulProcessor.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import jdk.incubator.vector.DoubleVector; +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorMask; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.DoubleBivariateLongDoubleFunction; + +import java.util.Arrays; + +/** + * SIMD specialization of {@code (long[], double[]) -> double[]} multiplication. The op is hardcoded to + * {@link DoubleVector#mul} so the JIT statically resolves it to the platform's double-multiply intrinsic. + */ +public final class SimdLongDoubleMulProcessor extends SimdLongDoubleProcessor +{ + public SimdLongDoubleMulProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + DoubleBivariateLongDoubleFunction scalarFallback + ) + { + super(left, right, scalarFallback); + } + + @Override + protected void processVector( + long[] leftInput, + double[] rightInput, + boolean[] leftNulls, + boolean[] rightNulls, + int currentSize + ) + { + final boolean hasLeftNulls = leftNulls != null; + final boolean hasRightNulls = rightNulls != null; + final int laneCount = DOUBLE_SPECIES.length(); + final int upperBound = DOUBLE_SPECIES.loopBound(currentSize); + int i = 0; + if (!hasLeftNulls && !hasRightNulls) { + for (; i < upperBound; i += laneCount) { + final DoubleVector va = + (DoubleVector) LongVector.fromArray(LONG_SPECIES, leftInput, i).castShape(DOUBLE_SPECIES, 0); + final DoubleVector vb = DoubleVector.fromArray(DOUBLE_SPECIES, rightInput, i); + va.mul(vb).intoArray(outValues, i); + } + for (; i < currentSize; i++) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + Arrays.fill(outNulls, 0, currentSize, false); + } else { + for (; i < upperBound; i += laneCount) { + final VectorMask nm; + if (hasLeftNulls && hasRightNulls) { + nm = VectorMask.fromArray(DOUBLE_SPECIES, leftNulls, i) + .or(VectorMask.fromArray(DOUBLE_SPECIES, rightNulls, i)); + } else if (hasLeftNulls) { + nm = VectorMask.fromArray(DOUBLE_SPECIES, leftNulls, i); + } else { + nm = VectorMask.fromArray(DOUBLE_SPECIES, rightNulls, i); + } + final DoubleVector va = + (DoubleVector) LongVector.fromArray(LONG_SPECIES, leftInput, i).castShape(DOUBLE_SPECIES, 0); + final DoubleVector vb = DoubleVector.fromArray(DOUBLE_SPECIES, rightInput, i); + va.mul(vb).intoArray(outValues, i); + nm.intoArray(outNulls, i); + } + for (; i < currentSize; i++) { + final boolean isNull = (hasLeftNulls && leftNulls[i]) || (hasRightNulls && rightNulls[i]); + outNulls[i] = isNull; + if (!isNull) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + } + } + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongDoubleProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongDoubleProcessor.java new file mode 100644 index 000000000000..366354f82b2e --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongDoubleProcessor.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import jdk.incubator.vector.DoubleVector; +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorSpecies; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExpressionType; +import org.apache.druid.math.expr.vector.CastToTypeVectorProcessor; +import org.apache.druid.math.expr.vector.ExprEvalDoubleVector; +import org.apache.druid.math.expr.vector.ExprEvalVector; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.DoubleBivariateLongDoubleFunction; + +import javax.annotation.Nullable; + +/** + * Abstract base for SIMD processors that compute {@code (long[], double[]) -> double[]} ops. The long lane is + * widened to {@link DoubleVector} via {@code castShape(DoubleVector.SPECIES_PREFERRED, 0)} in each subclass's hot + * loop. See {@link SimdLongLongProcessor} for the design rationale. + */ +abstract class SimdLongDoubleProcessor implements ExprVectorProcessor +{ + static final VectorSpecies LONG_SPECIES = LongVector.SPECIES_PREFERRED; + static final VectorSpecies DOUBLE_SPECIES = DoubleVector.SPECIES_PREFERRED; + + private final ExprVectorProcessor left; + private final ExprVectorProcessor right; + final DoubleBivariateLongDoubleFunction scalarFallback; + final double[] outValues; + final boolean[] outNulls; + + protected SimdLongDoubleProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + DoubleBivariateLongDoubleFunction scalarFallback + ) + { + this.left = CastToTypeVectorProcessor.cast(left, ExpressionType.LONG); + this.right = CastToTypeVectorProcessor.cast(right, ExpressionType.DOUBLE); + this.scalarFallback = scalarFallback; + this.outValues = new double[this.left.maxVectorSize()]; + this.outNulls = new boolean[this.left.maxVectorSize()]; + } + + @Override + public final ExprEvalVector evalVector(Expr.VectorInputBinding bindings) + { + final ExprEvalVector lhs = left.evalVector(bindings); + final ExprEvalVector rhs = right.evalVector(bindings); + processVector( + lhs.values(), + rhs.values(), + lhs.getNullVector(), + rhs.getNullVector(), + bindings.getCurrentVectorSize() + ); + return new ExprEvalDoubleVector(outValues, outNulls); + } + + protected abstract void processVector( + long[] leftInput, + double[] rightInput, + @Nullable boolean[] leftNulls, + @Nullable boolean[] rightNulls, + int currentSize + ); + + @Override + public final ExpressionType getOutputType() + { + return ExpressionType.DOUBLE; + } + + @Override + public final int maxVectorSize() + { + return outValues.length; + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongDoubleSubProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongDoubleSubProcessor.java new file mode 100644 index 000000000000..33c8602cf884 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongDoubleSubProcessor.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import jdk.incubator.vector.DoubleVector; +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorMask; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.DoubleBivariateLongDoubleFunction; + +import java.util.Arrays; + +/** + * SIMD specialization of {@code (long[], double[]) -> double[]} subtraction. The op is hardcoded to + * {@link DoubleVector#sub} so the JIT statically resolves it to the platform's double-subtract intrinsic. + */ +public final class SimdLongDoubleSubProcessor extends SimdLongDoubleProcessor +{ + public SimdLongDoubleSubProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + DoubleBivariateLongDoubleFunction scalarFallback + ) + { + super(left, right, scalarFallback); + } + + @Override + protected void processVector( + long[] leftInput, + double[] rightInput, + boolean[] leftNulls, + boolean[] rightNulls, + int currentSize + ) + { + final boolean hasLeftNulls = leftNulls != null; + final boolean hasRightNulls = rightNulls != null; + final int laneCount = DOUBLE_SPECIES.length(); + final int upperBound = DOUBLE_SPECIES.loopBound(currentSize); + int i = 0; + if (!hasLeftNulls && !hasRightNulls) { + for (; i < upperBound; i += laneCount) { + final DoubleVector va = + (DoubleVector) LongVector.fromArray(LONG_SPECIES, leftInput, i).castShape(DOUBLE_SPECIES, 0); + final DoubleVector vb = DoubleVector.fromArray(DOUBLE_SPECIES, rightInput, i); + va.sub(vb).intoArray(outValues, i); + } + for (; i < currentSize; i++) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + Arrays.fill(outNulls, 0, currentSize, false); + } else { + for (; i < upperBound; i += laneCount) { + final VectorMask nm; + if (hasLeftNulls && hasRightNulls) { + nm = VectorMask.fromArray(DOUBLE_SPECIES, leftNulls, i) + .or(VectorMask.fromArray(DOUBLE_SPECIES, rightNulls, i)); + } else if (hasLeftNulls) { + nm = VectorMask.fromArray(DOUBLE_SPECIES, leftNulls, i); + } else { + nm = VectorMask.fromArray(DOUBLE_SPECIES, rightNulls, i); + } + final DoubleVector va = + (DoubleVector) LongVector.fromArray(LONG_SPECIES, leftInput, i).castShape(DOUBLE_SPECIES, 0); + final DoubleVector vb = DoubleVector.fromArray(DOUBLE_SPECIES, rightInput, i); + va.sub(vb).intoArray(outValues, i); + nm.intoArray(outNulls, i); + } + for (; i < currentSize; i++) { + final boolean isNull = (hasLeftNulls && leftNulls[i]) || (hasRightNulls && rightNulls[i]); + outNulls[i] = isNull; + if (!isNull) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + } + } + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongLongAddProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongLongAddProcessor.java new file mode 100644 index 000000000000..f5e0298af09e --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongLongAddProcessor.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorMask; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.LongBivariateLongsFunction; + +import java.util.Arrays; + +/** + * SIMD specialization of {@code (long[], long[]) -> long[]} addition. The op is hardcoded to {@link LongVector#add} + * so the JIT statically resolves it to the platform's long-add intrinsic. + */ +public final class SimdLongLongAddProcessor extends SimdLongLongProcessor +{ + public SimdLongLongAddProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + LongBivariateLongsFunction scalarFallback + ) + { + super(left, right, scalarFallback); + } + + @Override + protected void processVector( + long[] leftInput, + long[] rightInput, + boolean[] leftNulls, + boolean[] rightNulls, + int currentSize + ) + { + final boolean hasLeftNulls = leftNulls != null; + final boolean hasRightNulls = rightNulls != null; + final int laneCount = SPECIES.length(); + final int upperBound = SPECIES.loopBound(currentSize); + int i = 0; + if (!hasLeftNulls && !hasRightNulls) { + for (; i < upperBound; i += laneCount) { + final LongVector va = LongVector.fromArray(SPECIES, leftInput, i); + final LongVector vb = LongVector.fromArray(SPECIES, rightInput, i); + va.add(vb).intoArray(outValues, i); + } + for (; i < currentSize; i++) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + Arrays.fill(outNulls, 0, currentSize, false); + } else { + for (; i < upperBound; i += laneCount) { + final VectorMask nm; + if (hasLeftNulls && hasRightNulls) { + nm = VectorMask.fromArray(SPECIES, leftNulls, i) + .or(VectorMask.fromArray(SPECIES, rightNulls, i)); + } else if (hasLeftNulls) { + nm = VectorMask.fromArray(SPECIES, leftNulls, i); + } else { + nm = VectorMask.fromArray(SPECIES, rightNulls, i); + } + final LongVector va = LongVector.fromArray(SPECIES, leftInput, i); + final LongVector vb = LongVector.fromArray(SPECIES, rightInput, i); + va.add(vb).intoArray(outValues, i); + nm.intoArray(outNulls, i); + } + for (; i < currentSize; i++) { + final boolean isNull = (hasLeftNulls && leftNulls[i]) || (hasRightNulls && rightNulls[i]); + outNulls[i] = isNull; + if (!isNull) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + } + } + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongLongMulProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongLongMulProcessor.java new file mode 100644 index 000000000000..32e8e8aa751b --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongLongMulProcessor.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorMask; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.LongBivariateLongsFunction; + +import java.util.Arrays; + +/** + * SIMD specialization of {@code (long[], long[]) -> long[]} multiplication. The op is hardcoded to + * {@link LongVector#mul} so the JIT statically resolves it to the platform's long-multiply intrinsic. + */ +public final class SimdLongLongMulProcessor extends SimdLongLongProcessor +{ + public SimdLongLongMulProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + LongBivariateLongsFunction scalarFallback + ) + { + super(left, right, scalarFallback); + } + + @Override + protected void processVector( + long[] leftInput, + long[] rightInput, + boolean[] leftNulls, + boolean[] rightNulls, + int currentSize + ) + { + final boolean hasLeftNulls = leftNulls != null; + final boolean hasRightNulls = rightNulls != null; + final int laneCount = SPECIES.length(); + final int upperBound = SPECIES.loopBound(currentSize); + int i = 0; + if (!hasLeftNulls && !hasRightNulls) { + for (; i < upperBound; i += laneCount) { + final LongVector va = LongVector.fromArray(SPECIES, leftInput, i); + final LongVector vb = LongVector.fromArray(SPECIES, rightInput, i); + va.mul(vb).intoArray(outValues, i); + } + for (; i < currentSize; i++) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + Arrays.fill(outNulls, 0, currentSize, false); + } else { + for (; i < upperBound; i += laneCount) { + final VectorMask nm; + if (hasLeftNulls && hasRightNulls) { + nm = VectorMask.fromArray(SPECIES, leftNulls, i) + .or(VectorMask.fromArray(SPECIES, rightNulls, i)); + } else if (hasLeftNulls) { + nm = VectorMask.fromArray(SPECIES, leftNulls, i); + } else { + nm = VectorMask.fromArray(SPECIES, rightNulls, i); + } + final LongVector va = LongVector.fromArray(SPECIES, leftInput, i); + final LongVector vb = LongVector.fromArray(SPECIES, rightInput, i); + va.mul(vb).intoArray(outValues, i); + nm.intoArray(outNulls, i); + } + for (; i < currentSize; i++) { + final boolean isNull = (hasLeftNulls && leftNulls[i]) || (hasRightNulls && rightNulls[i]); + outNulls[i] = isNull; + if (!isNull) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + } + } + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongLongProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongLongProcessor.java new file mode 100644 index 000000000000..999f4149fac2 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongLongProcessor.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorSpecies; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExpressionType; +import org.apache.druid.math.expr.vector.CastToTypeVectorProcessor; +import org.apache.druid.math.expr.vector.ExprEvalLongVector; +import org.apache.druid.math.expr.vector.ExprEvalVector; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.LongBivariateLongsFunction; + +import javax.annotation.Nullable; + +/** + * Abstract base for SIMD processors that compute {@code (long[], long[]) -> long[]} ops. Each concrete subclass + * (one per op) overrides {@link #processVector} with a hot loop that calls a statically-resolved {@link LongVector} + * method (e.g. {@code va.add(vb)}) so the JIT emits the corresponding SIMD intrinsic. + */ +abstract class SimdLongLongProcessor implements ExprVectorProcessor +{ + static final VectorSpecies SPECIES = LongVector.SPECIES_PREFERRED; + + private final ExprVectorProcessor left; + private final ExprVectorProcessor right; + final LongBivariateLongsFunction scalarFallback; + final long[] outValues; + final boolean[] outNulls; + + protected SimdLongLongProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + LongBivariateLongsFunction scalarFallback + ) + { + this.left = CastToTypeVectorProcessor.cast(left, ExpressionType.LONG); + this.right = CastToTypeVectorProcessor.cast(right, ExpressionType.LONG); + this.scalarFallback = scalarFallback; + this.outValues = new long[this.left.maxVectorSize()]; + this.outNulls = new boolean[this.left.maxVectorSize()]; + } + + @Override + public final ExprEvalVector evalVector(Expr.VectorInputBinding bindings) + { + final ExprEvalVector lhs = left.evalVector(bindings); + final ExprEvalVector rhs = right.evalVector(bindings); + processVector( + lhs.values(), + rhs.values(), + lhs.getNullVector(), + rhs.getNullVector(), + bindings.getCurrentVectorSize() + ); + return new ExprEvalLongVector(outValues, outNulls); + } + + protected abstract void processVector( + long[] leftInput, + long[] rightInput, + @Nullable boolean[] leftNulls, + @Nullable boolean[] rightNulls, + int currentSize + ); + + @Override + public final ExpressionType getOutputType() + { + return ExpressionType.LONG; + } + + @Override + public final int maxVectorSize() + { + return outValues.length; + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongLongSubProcessor.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongLongSubProcessor.java new file mode 100644 index 000000000000..ab85396463dd --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdLongLongSubProcessor.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorMask; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.LongBivariateLongsFunction; + +import java.util.Arrays; + +/** + * SIMD specialization of {@code (long[], long[]) -> long[]} subtraction. The op is hardcoded to {@link LongVector#sub} + * so the JIT statically resolves it to the platform's long-subtract intrinsic. + */ +public final class SimdLongLongSubProcessor extends SimdLongLongProcessor +{ + public SimdLongLongSubProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + LongBivariateLongsFunction scalarFallback + ) + { + super(left, right, scalarFallback); + } + + @Override + protected void processVector( + long[] leftInput, + long[] rightInput, + boolean[] leftNulls, + boolean[] rightNulls, + int currentSize + ) + { + final boolean hasLeftNulls = leftNulls != null; + final boolean hasRightNulls = rightNulls != null; + final int laneCount = SPECIES.length(); + final int upperBound = SPECIES.loopBound(currentSize); + int i = 0; + if (!hasLeftNulls && !hasRightNulls) { + for (; i < upperBound; i += laneCount) { + final LongVector va = LongVector.fromArray(SPECIES, leftInput, i); + final LongVector vb = LongVector.fromArray(SPECIES, rightInput, i); + va.sub(vb).intoArray(outValues, i); + } + for (; i < currentSize; i++) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + Arrays.fill(outNulls, 0, currentSize, false); + } else { + for (; i < upperBound; i += laneCount) { + final VectorMask nm; + if (hasLeftNulls && hasRightNulls) { + nm = VectorMask.fromArray(SPECIES, leftNulls, i) + .or(VectorMask.fromArray(SPECIES, rightNulls, i)); + } else if (hasLeftNulls) { + nm = VectorMask.fromArray(SPECIES, leftNulls, i); + } else { + nm = VectorMask.fromArray(SPECIES, rightNulls, i); + } + final LongVector va = LongVector.fromArray(SPECIES, leftInput, i); + final LongVector vb = LongVector.fromArray(SPECIES, rightInput, i); + va.sub(vb).intoArray(outValues, i); + nm.intoArray(outNulls, i); + } + for (; i < currentSize; i++) { + final boolean isNull = (hasLeftNulls && leftNulls[i]) || (hasRightNulls && rightNulls[i]); + outNulls[i] = isNull; + if (!isNull) { + outValues[i] = scalarFallback.process(leftInput[i], rightInput[i]); + } + } + } + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdProcessors.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdProcessors.java new file mode 100644 index 000000000000..d8d74021c7a0 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdProcessors.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +import org.apache.druid.error.DruidException; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.apache.druid.math.expr.vector.functional.DoubleBivariateDoubleLongFunction; +import org.apache.druid.math.expr.vector.functional.DoubleBivariateDoublesFunction; +import org.apache.druid.math.expr.vector.functional.DoubleBivariateLongDoubleFunction; +import org.apache.druid.math.expr.vector.functional.LongBivariateLongsFunction; + +/** + * Dispatch table from a {@link SimdSupportedBinaryOp} identifier to a concrete, op-specialized SIMD processor. + * One class per op and type-combo so the JIT sees a monomorphic call site for the SIMD operation in each hot loop. + */ +public final class SimdProcessors +{ + private SimdProcessors() + { + } + + public static ExprVectorProcessor makeLongLong( + ExprVectorProcessor left, + ExprVectorProcessor right, + SimdSupportedBinaryOp op, + LongBivariateLongsFunction scalarFallback + ) + { + return switch (op) { + case ADD -> new SimdLongLongAddProcessor(left, right, scalarFallback); + case SUB -> new SimdLongLongSubProcessor(left, right, scalarFallback); + case MUL -> new SimdLongLongMulProcessor(left, right, scalarFallback); + default -> throw DruidException.defensive("Unsupported SIMD binary op[%s]", op); + }; + } + + public static ExprVectorProcessor makeDoubleDouble( + ExprVectorProcessor left, + ExprVectorProcessor right, + SimdSupportedBinaryOp op, + DoubleBivariateDoublesFunction scalarFallback + ) + { + return switch (op) { + case ADD -> new SimdDoubleDoubleAddProcessor(left, right, scalarFallback); + case SUB -> new SimdDoubleDoubleSubProcessor(left, right, scalarFallback); + case MUL -> new SimdDoubleDoubleMulProcessor(left, right, scalarFallback); + default -> throw DruidException.defensive("Unsupported SIMD binary op[%s]", op); + }; + } + + public static ExprVectorProcessor makeLongDouble( + ExprVectorProcessor left, + ExprVectorProcessor right, + SimdSupportedBinaryOp op, + DoubleBivariateLongDoubleFunction scalarFallback + ) + { + return switch (op) { + case ADD -> new SimdLongDoubleAddProcessor(left, right, scalarFallback); + case SUB -> new SimdLongDoubleSubProcessor(left, right, scalarFallback); + case MUL -> new SimdLongDoubleMulProcessor(left, right, scalarFallback); + default -> throw DruidException.defensive("Unsupported SIMD binary op[%s]", op); + }; + } + + public static ExprVectorProcessor makeDoubleLong( + ExprVectorProcessor left, + ExprVectorProcessor right, + SimdSupportedBinaryOp op, + DoubleBivariateDoubleLongFunction scalarFallback + ) + { + return switch (op) { + case ADD -> new SimdDoubleLongAddProcessor(left, right, scalarFallback); + case SUB -> new SimdDoubleLongSubProcessor(left, right, scalarFallback); + case MUL -> new SimdDoubleLongMulProcessor(left, right, scalarFallback); + default -> throw DruidException.defensive("Unsupported SIMD binary op[%s]", op); + }; + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdSupportedBinaryOp.java b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdSupportedBinaryOp.java new file mode 100644 index 000000000000..953571ba4d36 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/simd/SimdSupportedBinaryOp.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector.simd; + +/** + * Identifies which binary math operations have a {@code jdk.incubator.vector} (SIMD) specialization. Used by + * {@link org.apache.druid.math.expr.vector.SimpleVectorMathBivariateProcessorFactory} subclasses to declare that + * their operation can be dispatched to a SIMD variant when the user enables + * {@link org.apache.druid.math.expr.ExpressionProcessingConfig#USE_VECTOR_API}. + * + * Deliberately does not reference any {@code jdk.incubator.vector} types so that callers wiring the enum into + * factories do not need the incubator module visible. + */ +public enum SimdSupportedBinaryOp +{ + ADD, + SUB, + MUL +} diff --git a/processing/src/test/java/org/apache/druid/math/expr/VectorExprResultConsistencyTest.java b/processing/src/test/java/org/apache/druid/math/expr/VectorExprResultConsistencyTest.java index 7f21e489f3d8..b837ca8ede31 100644 --- a/processing/src/test/java/org/apache/druid/math/expr/VectorExprResultConsistencyTest.java +++ b/processing/src/test/java/org/apache/druid/math/expr/VectorExprResultConsistencyTest.java @@ -67,7 +67,12 @@ public class VectorExprResultConsistencyTest extends InitializedNullHandlingTest { private static final Logger log = new Logger(VectorExprResultConsistencyTest.class); private static final int NUM_ITERATIONS = 10; - private static final int VECTOR_SIZE = 4; + /** + * Vector sizes used by each test iteration. Picked to exercise distinct regimes against typical SIMD species + * lengths: 3 is below {@code SPECIES.length()} on AVX2/AVX-512 (tail-only); 8 fills an exact AVX2/AVX-512 chunk; + * 17 leaves a ragged tail after one or more chunks; 67 spans many chunks plus a tail. + */ + private static final List VECTOR_SIZES = List.of(3, 8, 17, 67); private static final Map LOOKUP = Map.of( @@ -764,16 +769,18 @@ public static void testExpressionSequentialBindings( final int numIterations ) { - for (int iter = 0; iter < numIterations; iter++) { - assertEvalsMatch( - expr, - parsed, - makeSequentialBinding( - VECTOR_SIZE, - types, - -2 + (iter * VECTOR_SIZE) // include negative numbers and zero - ) - ); + for (int vectorSize : VECTOR_SIZES) { + for (int iter = 0; iter < numIterations; iter++) { + assertEvalsMatch( + expr, + parsed, + makeSequentialBinding( + vectorSize, + types, + -2 + (iter * vectorSize) // include negative numbers and zero + ) + ); + } } } @@ -784,8 +791,10 @@ public static void testExpressionRandomizedBindings( final int numIterations ) { - for (int iterations = 0; iterations < numIterations; iterations++) { - assertEvalsMatch(expr, parsed, makeRandomizedBindings(VECTOR_SIZE, types)); + for (int vectorSize : VECTOR_SIZES) { + for (int iterations = 0; iterations < numIterations; iterations++) { + assertEvalsMatch(expr, parsed, makeRandomizedBindings(vectorSize, types)); + } } } @@ -808,7 +817,8 @@ public static void assertEvalsMatch( ); if (vectorEval.isValue() && nonVectorEval.isValue()) { - for (int i = 0; i < VECTOR_SIZE; i++) { + final int vectorSize = bindings.lhs.length; + for (int i = 0; i < vectorSize; i++) { final String message = StringUtils.format( "Values do not match for row[%s] for expression[%s], bindings[%s]", i, @@ -1000,9 +1010,9 @@ private static Either evalNonVector( @Nullable ExpressionType outputType ) { - final Object[] exprValues = new Object[VECTOR_SIZE]; + final Object[] exprValues = new Object[bindings.length]; - for (int i = 0; i < VECTOR_SIZE; i++) { + for (int i = 0; i < bindings.length; i++) { ExprEval eval; try { eval = expr.eval(bindings[i]); diff --git a/processing/src/test/java/org/apache/druid/math/expr/VectorExprResultConsistencyVectorApiTest.java b/processing/src/test/java/org/apache/druid/math/expr/VectorExprResultConsistencyVectorApiTest.java new file mode 100644 index 000000000000..a2fc09996e91 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/math/expr/VectorExprResultConsistencyVectorApiTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr; + +import org.junit.After; +import org.junit.Before; + +/** + * Re-runs every {@link VectorExprResultConsistencyTest} case with the SIMD ({@code jdk.incubator.vector}) expression + * vector processors enabled, ensuring the SIMD specializations agree with the non-vectorized reference. Smoke-level + * coverage of the factory dispatch wiring; for deeper coverage of the SIMD chunk loop itself see + * {@link org.apache.druid.math.expr.vector.simd.SimdBivariateProcessorTest}. + */ +public class VectorExprResultConsistencyVectorApiTest extends VectorExprResultConsistencyTest +{ + @Before + public void enableVectorApi() + { + ExpressionProcessing.initializeForVectorApiTests(); + } + + @After + public void resetExpressionProcessing() + { + ExpressionProcessing.initializeForTests(); + } +} From 1ec1d9fe614dba1248205abc80a6c52ab8bfc4cc Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Sun, 24 May 2026 20:46:02 -0700 Subject: [PATCH 2/3] fix stale comment --- .../math/expr/VectorExprResultConsistencyVectorApiTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/math/expr/VectorExprResultConsistencyVectorApiTest.java b/processing/src/test/java/org/apache/druid/math/expr/VectorExprResultConsistencyVectorApiTest.java index a2fc09996e91..a14190796a27 100644 --- a/processing/src/test/java/org/apache/druid/math/expr/VectorExprResultConsistencyVectorApiTest.java +++ b/processing/src/test/java/org/apache/druid/math/expr/VectorExprResultConsistencyVectorApiTest.java @@ -24,9 +24,7 @@ /** * Re-runs every {@link VectorExprResultConsistencyTest} case with the SIMD ({@code jdk.incubator.vector}) expression - * vector processors enabled, ensuring the SIMD specializations agree with the non-vectorized reference. Smoke-level - * coverage of the factory dispatch wiring; for deeper coverage of the SIMD chunk loop itself see - * {@link org.apache.druid.math.expr.vector.simd.SimdBivariateProcessorTest}. + * vector processors enabled, ensuring the SIMD specializations agree with the non-vectorized reference. */ public class VectorExprResultConsistencyVectorApiTest extends VectorExprResultConsistencyTest { From 3c2c1eaf850a1fabd1ce7468985a18dd2b8a246d Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 25 May 2026 01:18:14 -0700 Subject: [PATCH 3/3] fixes --- benchmarks/pom.xml | 7 +++++++ .../query/SqlExpressionBenchmark.java | 20 +++++++++---------- pom.xml | 6 ++++++ .../expr/VectorExprResultConsistencyTest.java | 5 ----- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 12fec8bb4e72..c02c2166f115 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -255,6 +255,13 @@ org.openjdk.jmh.generators.BenchmarkProcessor + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java index 361eddff4949..0ef6395a1fce 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java @@ -177,16 +177,6 @@ public class SqlExpressionBenchmark extends SqlBaseQueryBenchmark @Param({"false", "true"}) private boolean useVectorApi; - @Setup(Level.Trial) - public void setupExpressionProcessing() - { - if (useVectorApi) { - ExpressionProcessing.initializeForVectorApiTests(); - } else { - ExpressionProcessing.initializeForTests(); - } - } - @Param({ // non-expression reference "0", @@ -254,6 +244,16 @@ public void setupExpressionProcessing() }) private String query; + @Setup(Level.Trial) + public void setupExpressionProcessing() + { + if (useVectorApi) { + ExpressionProcessing.initializeForVectorApiTests(); + } else { + ExpressionProcessing.initializeForTests(); + } + } + @Override public String getQuery() { diff --git a/pom.xml b/pom.xml index 1f0640b03a24..6202c59ed19c 100644 --- a/pom.xml +++ b/pom.xml @@ -2147,6 +2147,11 @@ org.apache.hadoop.fs + + + + --add-modules=jdk.incubator.vector + @@ -2220,6 +2225,7 @@ -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED -J--add-modules=jdk.incubator.vector + --add-modules=jdk.incubator.vector diff --git a/processing/src/test/java/org/apache/druid/math/expr/VectorExprResultConsistencyTest.java b/processing/src/test/java/org/apache/druid/math/expr/VectorExprResultConsistencyTest.java index b837ca8ede31..9ffe5da3ace8 100644 --- a/processing/src/test/java/org/apache/druid/math/expr/VectorExprResultConsistencyTest.java +++ b/processing/src/test/java/org/apache/druid/math/expr/VectorExprResultConsistencyTest.java @@ -67,11 +67,6 @@ public class VectorExprResultConsistencyTest extends InitializedNullHandlingTest { private static final Logger log = new Logger(VectorExprResultConsistencyTest.class); private static final int NUM_ITERATIONS = 10; - /** - * Vector sizes used by each test iteration. Picked to exercise distinct regimes against typical SIMD species - * lengths: 3 is below {@code SPECIES.length()} on AVX2/AVX-512 (tail-only); 8 fills an exact AVX2/AVX-512 chunk; - * 17 leaves a ragged tail after one or more chunks; 67 spans many chunks plus a tail. - */ private static final List VECTOR_SIZES = List.of(3, 8, 17, 67);