From 73a92e21f5627d0105b10f6eb7837b75ac96b801 Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Mon, 21 Oct 2024 16:01:42 +0530 Subject: [PATCH 1/7] 8342676: Unsigned Vector Min / Max transforms --- src/hotspot/share/opto/vectornode.cpp | 43 ++ src/hotspot/share/opto/vectornode.hpp | 18 + .../compiler/lib/ir_framework/IRNode.java | 40 ++ .../VectorUnsignedMinMaxOperationsTest.java | 391 ++++++++++++++++++ 4 files changed, 492 insertions(+) create mode 100644 test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 094d4dca564a8..f2b881e43b1e6 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -2078,6 +2078,49 @@ Node* VectorBlendNode::Identity(PhaseGVN* phase) { return this; } +Node* UMinVNode::Ideal(PhaseGVN* phase, bool can_reshape) { + bool match1 = in(1)->Opcode() == Op_UMinV || in(1)->Opcode() == Op_UMaxV; + bool match2 = in(2)->Opcode() == Op_UMinV || in(2)->Opcode() == Op_UMaxV; + // UMin (UMin(a, b), UMax(a, b)) => UMin(a, b) + // UMin (UMin(a, b), UMax(b, a)) => UMin(a, b) + if (match1 && match2) { + if ((in(1)->in(1) == in(2)->in(1) && in(1)->in(2) == in(2)->in(2)) || + (in(1)->in(2) == in(2)->in(1) && in(1)->in(1) == in(2)->in(2))) { + return new UMinVNode(in(1)->in(1), in(1)->in(2), vect_type()); + } + } + return nullptr; +} + +Node* UMinVNode::Identity(PhaseGVN* phase) { + // UMin (a, a) => a + if (in(1) == in(2)) { + return in(1); + } + return this; +} + +Node* UMaxVNode::Ideal(PhaseGVN* phase, bool can_reshape) { + bool match1 = in(1)->Opcode() == Op_UMinV || in(1)->Opcode() == Op_UMaxV; + bool match2 = in(2)->Opcode() == Op_UMinV || in(2)->Opcode() == Op_UMaxV; + // UMax (UMin(a, b), UMax(a, b)) => UMax(a, b) + // UMax (UMin(a, b), UMax(b, a)) => UMax(a, b) + if (match1 && match2) { + if ((in(1)->in(1) == in(2)->in(1) && in(1)->in(2) == in(2)->in(2)) || + (in(1)->in(2) == in(2)->in(1) && in(1)->in(1) == in(2)->in(2))) { + return new UMaxVNode(in(1)->in(1), in(1)->in(2), vect_type()); + } + } + return nullptr; +} + +Node* UMaxVNode::Identity(PhaseGVN* phase) { + // UMax (a, a) => a + if (in(1) == in(2)) { + return in(1); + } + return this; +} #ifndef PRODUCT void VectorBoxAllocateNode::dump_spec(outputStream *st) const { CallStaticJavaNode::dump_spec(st); diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index d23e6b8c9268d..21a045ae79211 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -1967,4 +1967,22 @@ class ExpandBitsVNode : public VectorNode { virtual int Opcode() const; }; +class UMinVNode : public VectorNode { + public: + UMinVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2 ,vt) { + assert(is_integral_type(vt->element_basic_type()), ""); + } + virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); + virtual Node* Identity(PhaseGVN* phase); + virtual int Opcode() const; +}; + + @@ -614,6 +626,8 @@ class UMaxVNode : public VectorNode { + UMaxVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) { + assert(is_integral_type(vt->element_basic_type()), ""); + } + virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); + virtual Node* Identity(PhaseGVN* phase); + virtual int Opcode() const; +}; #endif // SHARE_OPTO_VECTORNODE_HPP diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java index 8c4b3c93343f6..79b20d2b1e3a9 100644 --- a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java +++ b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java @@ -1133,6 +1133,46 @@ public class IRNode { vectorNode(MUL_ADD_VS2VI, "MulAddVS2VI", TYPE_INT); } + public static final String UMIN_VB = VECTOR_PREFIX + "UMIN_VB" + POSTFIX; + static { + vectorNode(UMIN_VB, "UMinV", TYPE_BYTE); + } + + public static final String UMIN_VS = VECTOR_PREFIX + "UMIN_VS" + POSTFIX; + static { + vectorNode(UMIN_VS, "UMinV", TYPE_SHORT); + } + + public static final String UMIN_VI = VECTOR_PREFIX + "UMIN_VI" + POSTFIX; + static { + vectorNode(UMIN_VI, "UMinV", TYPE_INT); + } + + public static final String UMIN_VL = VECTOR_PREFIX + "UMIN_VL" + POSTFIX; + static { + vectorNode(UMIN_VL, "UMinV", TYPE_LONG); + } + + public static final String UMAX_VB = VECTOR_PREFIX + "UMAX_VB" + POSTFIX; + static { + vectorNode(UMAX_VB, "UMaxV", TYPE_BYTE); + } + + public static final String UMAX_VS = VECTOR_PREFIX + "UMAX_VS" + POSTFIX; + static { + vectorNode(UMAX_VS, "UMaxV", TYPE_SHORT); + } + + public static final String UMAX_VI = VECTOR_PREFIX + "UMAX_VI" + POSTFIX; + static { + vectorNode(UMAX_VI, "UMaxV", TYPE_INT); + } + + public static final String UMAX_VL = VECTOR_PREFIX + "UMAX_VL" + POSTFIX; + static { + vectorNode(UMAX_VL, "UMaxV", TYPE_LONG); + } + // Can only be used if avx512_vnni is available. public static final String MUL_ADD_VS2VI_VNNI = PREFIX + "MUL_ADD_VS2VI_VNNI" + POSTFIX; static { diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java new file mode 100644 index 0000000000000..cceb40dfcffcf --- /dev/null +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java @@ -0,0 +1,391 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** +* @test +* @bug 8338201 +* @summary Support new unsigned and saturating vector operators in VectorAPI +* @modules jdk.incubator.vector +* @requires vm.compiler2.enabled +* @library /test/lib / +* @run driver compiler.vectorapi.VectorUnsignedMinMaxOperationsTest +*/ + +package compiler.vectorapi; + +import jdk.incubator.vector.*; +import compiler.lib.ir_framework.*; +import java.util.stream.IntStream; + +public class VectorUnsignedMinMaxOperationsTest { + private static final int COUNT = 2048; + private static final VectorSpecies lspec = LongVector.SPECIES_PREFERRED; + private static final VectorSpecies ispec = IntVector.SPECIES_PREFERRED; + private static final VectorSpecies sspec = ShortVector.SPECIES_PREFERRED; + private static final VectorSpecies bspec = ByteVector.SPECIES_PREFERRED; + + private long[] long_in1; + private int[] int_in1; + private short[] short_in1; + private byte[] byte_in1; + + private long[] long_in2; + private int[] int_in2; + private short[] short_in2; + private byte[] byte_in2; + + private long[] long_out; + private int[] int_out; + private short[] short_out; + private byte[] byte_out; + + public static void main(String[] args) { + TestFramework testFramework = new TestFramework(); + testFramework.setDefaultWarmup(5000) + .addFlags("--add-modules=jdk.incubator.vector") + .start(); + } + + public VectorUnsignedMinMaxOperationsTest() { + byte_in1 = new byte[COUNT]; + short_in1 = new short[COUNT]; + int_in1 = new int[COUNT]; + long_in1 = new long[COUNT]; + + byte_in2 = new byte[COUNT]; + short_in2 = new short[COUNT]; + int_in2 = new int[COUNT]; + long_in2 = new long[COUNT]; + IntStream.range(0, COUNT).forEach( + i -> { + if ((i & 1) == 0) { + long_in1[i] = Long.MAX_VALUE; + long_in2[i] = i; + int_in1[i] = Integer.MAX_VALUE; + int_in2[i] = i; + short_in1[i] = Short.MAX_VALUE; + short_in2[i] = (short)i; + byte_in1[i] = Byte.MAX_VALUE; + byte_in2[i] = (byte)i; + } else { + long_in1[i] = Long.MIN_VALUE; + long_in2[i] = -i; + int_in1[i] = Integer.MIN_VALUE; + int_in2[i] = -i; + short_in1[i] = Short.MIN_VALUE; + short_in2[i] = (short)-i; + byte_in1[i] = Byte.MIN_VALUE; + byte_in2[i] = (byte)-i; + } + } + ); + long_out = new long[COUNT]; + int_out = new int[COUNT]; + short_out = new short[COUNT]; + byte_out = new byte[COUNT]; + } + + @Test + @IR(counts = {IRNode.UMAX_VB, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @Warmup(value = 10000) + public void umax_byte() { + for (int i = 0; i < COUNT; i += bspec.length()) { + ByteVector.fromArray(bspec, byte_in1, i) + .lanewise(VectorOperators.UMAX, + ByteVector.fromArray(bspec, byte_in2, i)) + .intoArray(byte_out, i); + } + } + + @Check(test = "umax_byte") + public void umax_byte_verify() { + for (int i = 0; i < COUNT; i++) { + byte actual = byte_out[i]; + byte expected = VectorMath.maxUnsigned(byte_in1[i], byte_in2[i]); + if (actual != expected) { + throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); + } + } + } + + @Test + @IR(counts = {IRNode.UMAX_VS, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @Warmup(value = 10000) + public void umax_short() { + for (int i = 0; i < COUNT; i += sspec.length()) { + ShortVector.fromArray(sspec, short_in1, i) + .lanewise(VectorOperators.UMAX, + ShortVector.fromArray(sspec, short_in2, i)) + .intoArray(short_out, i); + } + } + + @Check(test = "umax_short") + public void umax_short_verify() { + for (int i = 0; i < COUNT; i++) { + short actual = short_out[i]; + short expected = VectorMath.maxUnsigned(short_in1[i], short_in2[i]); + if (actual != expected) { + throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); + } + } + } + + @Test + @IR(counts = {IRNode.UMAX_VI, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @Warmup(value = 10000) + public void umax_int() { + for (int i = 0; i < COUNT; i += ispec.length()) { + IntVector.fromArray(ispec, int_in1, i) + .lanewise(VectorOperators.UMAX, + IntVector.fromArray(ispec, int_in2, i)) + .intoArray(int_out, i); + } + } + + @Check(test = "umax_int") + public void umax_int_verify() { + for (int i = 0; i < COUNT; i++) { + int actual = int_out[i]; + int expected = VectorMath.maxUnsigned(int_in1[i], int_in2[i]); + if (actual != expected) { + throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); + } + } + } + + @Test + @IR(counts = {IRNode.UMAX_VL, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @Warmup(value = 10000) + public void umax_long() { + for (int i = 0; i < COUNT; i += lspec.length()) { + LongVector.fromArray(lspec, long_in1, i) + .lanewise(VectorOperators.UMAX, + LongVector.fromArray(lspec, long_in2, i)) + .intoArray(long_out, i); + } + } + + @Check(test = "umax_long") + public void umax_long_verify() { + for (int i = 0; i < COUNT; i++) { + long actual = long_out[i]; + long expected = VectorMath.maxUnsigned(long_in1[i], long_in2[i]); + if (actual != expected) { + throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); + } + } + } + + @Test + @IR(counts = {IRNode.UMIN_VB, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @Warmup(value = 10000) + public void umin_byte() { + for (int i = 0; i < COUNT; i += bspec.length()) { + ByteVector.fromArray(bspec, byte_in1, i) + .lanewise(VectorOperators.UMIN, + ByteVector.fromArray(bspec, byte_in2, i)) + .intoArray(byte_out, i); + } + } + + @Check(test = "umin_byte") + public void umin_byte_verify() { + for (int i = 0; i < COUNT; i++) { + byte actual = byte_out[i]; + byte expected = VectorMath.minUnsigned(byte_in1[i], byte_in2[i]); + if (actual != expected) { + throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); + } + } + } + + @Test + @IR(counts = {IRNode.UMIN_VS, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @Warmup(value = 10000) + public void umin_short() { + for (int i = 0; i < COUNT; i += sspec.length()) { + ShortVector.fromArray(sspec, short_in1, i) + .lanewise(VectorOperators.UMIN, + ShortVector.fromArray(sspec, short_in2, i)) + .intoArray(short_out, i); + } + } + + @Check(test = "umin_short") + public void umin_short_verify() { + for (int i = 0; i < COUNT; i++) { + short actual = short_out[i]; + short expected = VectorMath.minUnsigned(short_in1[i], short_in2[i]); + if (actual != expected) { + throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); + } + } + } + + @Test + @IR(counts = {IRNode.UMIN_VI, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @Warmup(value = 10000) + public void umin_int() { + for (int i = 0; i < COUNT; i += ispec.length()) { + IntVector.fromArray(ispec, int_in1, i) + .lanewise(VectorOperators.UMIN, + IntVector.fromArray(ispec, int_in2, i)) + .intoArray(int_out, i); + } + } + + @Check(test = "umin_int") + public void umin_int_verify() { + for (int i = 0; i < COUNT; i++) { + int actual = int_out[i]; + int expected = VectorMath.minUnsigned(int_in1[i], int_in2[i]); + if (actual != expected) { + throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); + } + } + } + + @Test + @IR(counts = {IRNode.UMIN_VL, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @Warmup(value = 10000) + public void umin_long() { + for (int i = 0; i < COUNT; i += lspec.length()) { + LongVector.fromArray(lspec, long_in1, i) + .lanewise(VectorOperators.UMIN, + LongVector.fromArray(lspec, long_in2, i)) + .intoArray(long_out, i); + } + } + + @Check(test = "umin_long") + public void umin_long_verify() { + for (int i = 0; i < COUNT; i++) { + long actual = long_out[i]; + long expected = VectorMath.minUnsigned(long_in1[i], long_in2[i]); + if (actual != expected) { + throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); + } + } + } + + @Test + @IR(counts = {IRNode.UMIN_VI, " 0 "}, applyIf = {"UseAVX", " >0 "}) + @Warmup(value = 10000) + public void umin_ir_transform1() { + for (int i = 0; i < COUNT; i += ispec.length()) { + IntVector.fromArray(ispec, int_in1, i) + .lanewise(VectorOperators.UMIN, + IntVector.fromArray(ispec, int_in1, i)) + .intoArray(int_out, i); + } + } + + @Check(test = "umin_ir_transform1") + public void umin_ir_transform1_verify() { + for (int i = 0; i < COUNT; i++) { + int actual = int_out[i]; + int expected = VectorMath.minUnsigned(int_in1[i], int_in1[i]); + if (actual != expected) { + throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); + } + } + } + + @Test + @IR(counts = {IRNode.UMAX_VI, " 0 "}, applyIf = {"UseAVX", " >0 "}) + @Warmup(value = 10000) + public void umax_ir_transform1() { + for (int i = 0; i < COUNT; i += ispec.length()) { + IntVector.fromArray(ispec, int_in1, i) + .lanewise(VectorOperators.UMAX, + IntVector.fromArray(ispec, int_in1, i)) + .intoArray(int_out, i); + } + } + + @Check(test = "umax_ir_transform1") + public void umax_ir_transform1_verify() { + for (int i = 0; i < COUNT; i++) { + int actual = int_out[i]; + int expected = VectorMath.maxUnsigned(int_in1[i], int_in1[i]); + if (actual != expected) { + throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); + } + } + } + + @Test + @IR(counts = {IRNode.UMAX_VI, " 0 ", IRNode.UMIN_VI, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @Warmup(value = 10000) + public void umin_max_ir_transform1() { + for (int i = 0; i < COUNT; i += ispec.length()) { + IntVector vec1 = IntVector.fromArray(ispec, int_in1, i); + IntVector vec2 = IntVector.fromArray(ispec, int_in2, i); + // UMinV (UMinV vec1, vec2) (UMaxV vec1, vec2) => UMinV vec1 vec2 + vec1.lanewise(VectorOperators.UMIN, vec2) + .lanewise(VectorOperators.UMIN, + vec1.lanewise(VectorOperators.UMAX, vec2)) + .intoArray(int_out, i); + } + } + + @Check(test = "umin_max_ir_transform1") + public void umin_max_ir_transform1_verify() { + for (int i = 0; i < COUNT; i++) { + int actual = int_out[i]; + int expected = VectorMath.minUnsigned(VectorMath.minUnsigned(int_in1[i], int_in2[i]), + VectorMath.maxUnsigned(int_in1[i], int_in2[i])); + if (actual != expected) { + throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); + } + } + } + + @Test + @IR(counts = {IRNode.UMIN_VI, " 0 ", IRNode.UMAX_VI, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @Warmup(value = 10000) + public void umin_max_ir_transform2() { + for (int i = 0; i < COUNT; i += ispec.length()) { + IntVector vec1 = IntVector.fromArray(ispec, int_in1, i); + IntVector vec2 = IntVector.fromArray(ispec, int_in2, i); + // UMinV (UMinV vec1, vec2) (UMaxV vec1, vec2) => UMinV vec1 vec2 + vec1.lanewise(VectorOperators.UMIN, vec2) + .lanewise(VectorOperators.UMAX, + vec1.lanewise(VectorOperators.UMAX, vec2)) + .intoArray(int_out, i); + } + } + + @Check(test = "umin_max_ir_transform2") + public void umin_max_ir_transform2_verify() { + for (int i = 0; i < COUNT; i++) { + int actual = int_out[i]; + int expected = VectorMath.maxUnsigned(VectorMath.minUnsigned(int_in1[i], int_in2[i]), + VectorMath.maxUnsigned(int_in1[i], int_in2[i])); + if (actual != expected) { + throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); + } + } + } +} From 7c03580204e5fb971b9f0bb23a8a9b660c064f56 Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Mon, 23 Dec 2024 06:12:37 -0500 Subject: [PATCH 2/7] Update IR transforms and tests --- src/hotspot/share/opto/vectornode.cpp | 54 +++++---- src/hotspot/share/opto/vectornode.hpp | 29 ++--- .../VectorUnsignedMinMaxOperationsTest.java | 103 ++++++++++++++++-- 3 files changed, 138 insertions(+), 48 deletions(-) diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index bb1829c881c6d..dfa4c4cc81381 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -2132,20 +2132,46 @@ bool MulVLNode::has_uint_inputs() const { has_vector_elements_fit_uint(in(2)); } -Node* UMinVNode::Ideal(PhaseGVN* phase, bool can_reshape) { - bool match1 = in(1)->Opcode() == Op_UMinV || in(1)->Opcode() == Op_UMaxV; - bool match2 = in(2)->Opcode() == Op_UMinV || in(2)->Opcode() == Op_UMaxV; +static Node* unsigned_min_max_xform(Node* n) { + int vopc = n->Opcode(); + assert(vopc == Op_UMinV || vopc == Op_UMaxV, "Unexpected opcode"); + + Node* umin = nullptr; + Node* umax = nullptr; + int lopc = n->in(1)->Opcode(); + int ropc = n->in(2)->Opcode(); + + if (lopc == Op_UMinV && ropc == Op_UMaxV) { + umin = n->in(1); + umax = n->in(2); + } + else if (lopc == Op_UMaxV && ropc == Op_UMinV) { + umin = n->in(2); + umax = n->in(1); + } + // UMin (UMin(a, b), UMax(a, b)) => UMin(a, b) - // UMin (UMin(a, b), UMax(b, a)) => UMin(a, b) - if (match1 && match2) { - if ((in(1)->in(1) == in(2)->in(1) && in(1)->in(2) == in(2)->in(2)) || - (in(1)->in(2) == in(2)->in(1) && in(1)->in(1) == in(2)->in(2))) { - return new UMinVNode(in(1)->in(1), in(1)->in(2), vect_type()); + // UMin (UMax(a, b), UMin(b, a)) => UMin(a, b) + // UMax (UMin(a, b), UMax(a, b)) => UMax(a, b) + // UMax (UMax(a, b), UMin(b, a)) => UMax(a, b) + if (umin && umax) { + if ((umin->in(1) == umax->in(1) && umin->in(2) == umax->in(2)) || + (umin->in(2) == umax->in(1) && umin->in(1) == umax->in(2))) { + if (vopc == Op_UMinV) { + return new UMinVNode(umax->in(1), umax->in(2), n->bottom_type()->is_vect()); + } else { + return new UMaxVNode(umax->in(1), umax->in(2), n->bottom_type()->is_vect()); + } } } + return nullptr; } +Node* UMinVNode::Ideal(PhaseGVN* phase, bool can_reshape) { + return unsigned_min_max_xform(this); +} + Node* UMinVNode::Identity(PhaseGVN* phase) { // UMin (a, a) => a if (in(1) == in(2)) { @@ -2155,17 +2181,7 @@ Node* UMinVNode::Identity(PhaseGVN* phase) { } Node* UMaxVNode::Ideal(PhaseGVN* phase, bool can_reshape) { - bool match1 = in(1)->Opcode() == Op_UMinV || in(1)->Opcode() == Op_UMaxV; - bool match2 = in(2)->Opcode() == Op_UMinV || in(2)->Opcode() == Op_UMaxV; - // UMax (UMin(a, b), UMax(a, b)) => UMax(a, b) - // UMax (UMin(a, b), UMax(b, a)) => UMax(a, b) - if (match1 && match2) { - if ((in(1)->in(1) == in(2)->in(1) && in(1)->in(2) == in(2)->in(2)) || - (in(1)->in(2) == in(2)->in(1) && in(1)->in(1) == in(2)->in(2))) { - return new UMaxVNode(in(1)->in(1), in(1)->in(2), vect_type()); - } - } - return nullptr; + return unsigned_min_max_xform(this); } Node* UMaxVNode::Identity(PhaseGVN* phase) { diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index cb60ae546e07f..36407b19b3f3d 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -612,9 +612,15 @@ class UMinVNode : public VectorNode { UMinVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2 ,vt) { assert(is_integral_type(vt->element_basic_type()), ""); } + virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); + virtual Node* Identity(PhaseGVN* phase); virtual int Opcode() const; + virtual uint hash() const { + return (uintptr_t)in(1) + (uintptr_t)in(2) + Opcode(); + } }; + //------------------------------MaxVNode-------------------------------------- // Vector Max class MaxVNode : public VectorNode { @@ -628,7 +634,12 @@ class UMaxVNode : public VectorNode { UMaxVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) { assert(is_integral_type(vt->element_basic_type()), ""); } + virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); + virtual Node* Identity(PhaseGVN* phase); virtual int Opcode() const; + virtual uint hash() const { + return (uintptr_t)in(1) + (uintptr_t)in(2) + Opcode(); + } }; //------------------------------AbsVINode-------------------------------------- @@ -2043,22 +2054,4 @@ class ExpandBitsVNode : public VectorNode { virtual int Opcode() const; }; -class UMinVNode : public VectorNode { - public: - UMinVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2 ,vt) { - assert(is_integral_type(vt->element_basic_type()), ""); - } - virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); - virtual Node* Identity(PhaseGVN* phase); - virtual int Opcode() const; -}; - - @@ -614,6 +626,8 @@ class UMaxVNode : public VectorNode { - UMaxVNode(Node* in1, Node* in2, const TypeVect* vt) : VectorNode(in1, in2, vt) { - assert(is_integral_type(vt->element_basic_type()), ""); - } - virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); - virtual Node* Identity(PhaseGVN* phase); - virtual int Opcode() const; -}; #endif // SHARE_OPTO_VECTORNODE_HPP diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java index cceb40dfcffcf..c2a1cdd812343 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java @@ -117,7 +117,7 @@ public void umax_byte() { } } - @Check(test = "umax_byte") + @Check(test = "umax_byte", when = CheckAt.COMPILED) public void umax_byte_verify() { for (int i = 0; i < COUNT; i++) { byte actual = byte_out[i]; @@ -140,7 +140,7 @@ public void umax_short() { } } - @Check(test = "umax_short") + @Check(test = "umax_short", when = CheckAt.COMPILED) public void umax_short_verify() { for (int i = 0; i < COUNT; i++) { short actual = short_out[i]; @@ -163,7 +163,7 @@ public void umax_int() { } } - @Check(test = "umax_int") + @Check(test = "umax_int", when = CheckAt.COMPILED) public void umax_int_verify() { for (int i = 0; i < COUNT; i++) { int actual = int_out[i]; @@ -186,7 +186,7 @@ public void umax_long() { } } - @Check(test = "umax_long") + @Check(test = "umax_long", when = CheckAt.COMPILED) public void umax_long_verify() { for (int i = 0; i < COUNT; i++) { long actual = long_out[i]; @@ -209,7 +209,7 @@ public void umin_byte() { } } - @Check(test = "umin_byte") + @Check(test = "umin_byte", when = CheckAt.COMPILED) public void umin_byte_verify() { for (int i = 0; i < COUNT; i++) { byte actual = byte_out[i]; @@ -232,7 +232,7 @@ public void umin_short() { } } - @Check(test = "umin_short") + @Check(test = "umin_short", when = CheckAt.COMPILED) public void umin_short_verify() { for (int i = 0; i < COUNT; i++) { short actual = short_out[i]; @@ -255,7 +255,7 @@ public void umin_int() { } } - @Check(test = "umin_int") + @Check(test = "umin_int", when = CheckAt.COMPILED) public void umin_int_verify() { for (int i = 0; i < COUNT; i++) { int actual = int_out[i]; @@ -301,7 +301,7 @@ public void umin_ir_transform1() { } } - @Check(test = "umin_ir_transform1") + @Check(test = "umin_ir_transform1", when = CheckAt.COMPILED) public void umin_ir_transform1_verify() { for (int i = 0; i < COUNT; i++) { int actual = int_out[i]; @@ -324,7 +324,7 @@ public void umax_ir_transform1() { } } - @Check(test = "umax_ir_transform1") + @Check(test = "umax_ir_transform1", when = CheckAt.COMPILED) public void umax_ir_transform1_verify() { for (int i = 0; i < COUNT; i++) { int actual = int_out[i]; @@ -350,7 +350,7 @@ public void umin_max_ir_transform1() { } } - @Check(test = "umin_max_ir_transform1") + @Check(test = "umin_max_ir_transform1", when = CheckAt.COMPILED) public void umin_max_ir_transform1_verify() { for (int i = 0; i < COUNT; i++) { int actual = int_out[i]; @@ -377,7 +377,7 @@ public void umin_max_ir_transform2() { } } - @Check(test = "umin_max_ir_transform2") + @Check(test = "umin_max_ir_transform2", when = CheckAt.COMPILED) public void umin_max_ir_transform2_verify() { for (int i = 0; i < COUNT; i++) { int actual = int_out[i]; @@ -388,4 +388,85 @@ public void umin_max_ir_transform2_verify() { } } } + + @Test + @IR(counts = {IRNode.UMAX_VI, " 0 ", IRNode.UMIN_VI, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @Warmup(value = 10000) + public void umin_max_ir_transform3() { + for (int i = 0; i < COUNT; i += ispec.length()) { + IntVector vec1 = IntVector.fromArray(ispec, int_in1, i); + IntVector vec2 = IntVector.fromArray(ispec, int_in2, i); + // UMaxV (UMinV vec1, vec2) (UMinV vec1, vec2) => UMinV vec1 vec2 + vec1.lanewise(VectorOperators.UMIN, vec2) + .lanewise(VectorOperators.UMAX, + vec1.lanewise(VectorOperators.UMIN, vec2)) + .intoArray(int_out, i); + } + } + + @Check(test = "umin_max_ir_transform3", when = CheckAt.COMPILED) + public void umin_max_ir_transform3_verify() { + for (int i = 0; i < COUNT; i++) { + int actual = int_out[i]; + int expected = VectorMath.maxUnsigned(VectorMath.minUnsigned(int_in1[i], int_in2[i]), + VectorMath.minUnsigned(int_in1[i], int_in2[i])); + if (actual != expected) { + throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); + } + } + } + + @Test + @IR(counts = {IRNode.UMIN_VI, " 0 ", IRNode.UMAX_VI, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @Warmup(value = 10000) + public void umin_max_ir_transform4() { + for (int i = 0; i < COUNT; i += ispec.length()) { + IntVector vec1 = IntVector.fromArray(ispec, int_in1, i); + IntVector vec2 = IntVector.fromArray(ispec, int_in2, i); + // UMinV (UMaxV vec1, vec2) (UMaxV vec1, vec2) => UMaxV vec1 vec2 + vec1.lanewise(VectorOperators.UMAX, vec2) + .lanewise(VectorOperators.UMIN, + vec1.lanewise(VectorOperators.UMAX, vec2)) + .intoArray(int_out, i); + } + } + + @Check(test = "umin_max_ir_transform4", when = CheckAt.COMPILED) + public void umin_max_ir_transform4_verify() { + for (int i = 0; i < COUNT; i++) { + int actual = int_out[i]; + int expected = VectorMath.minUnsigned(VectorMath.maxUnsigned(int_in1[i], int_in2[i]), + VectorMath.maxUnsigned(int_in1[i], int_in2[i])); + if (actual != expected) { + throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); + } + } + } + + @Test + @IR(counts = {IRNode.UMIN_VI, " 0 ", IRNode.UMAX_VI, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @Warmup(value = 10000) + public void umin_max_ir_transform5() { + for (int i = 0; i < COUNT; i += ispec.length()) { + IntVector vec1 = IntVector.fromArray(ispec, int_in1, i); + IntVector vec2 = IntVector.fromArray(ispec, int_in2, i); + // UMinV (UMinV vec1, vec2) (UMaxV vec1, vec2) => UMinV vec1 vec2 + vec1.lanewise(VectorOperators.UMIN, vec2) + .lanewise(VectorOperators.UMAX, + vec1.lanewise(VectorOperators.UMAX, vec2)) + .intoArray(int_out, i); + } + } + + @Check(test = "umin_max_ir_transform5", when = CheckAt.COMPILED) + public void umin_max_ir_transform5_verify() { + for (int i = 0; i < COUNT; i++) { + int actual = int_out[i]; + int expected = VectorMath.maxUnsigned(VectorMath.minUnsigned(int_in1[i], int_in2[i]), + VectorMath.maxUnsigned(int_in1[i], int_in2[i])); + if (actual != expected) { + throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); + } + } + } } From cc39220a58919651211bcfee78b6be67a66fd8d8 Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Tue, 7 Jan 2025 14:02:04 +0530 Subject: [PATCH 3/7] Updating copyright year of modified files --- test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java | 2 +- .../compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java index 4016f02ae66a4..58733f3d7ece1 100644 --- a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java +++ b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java index c2a1cdd812343..c77c321db95a6 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From e9e09a5b733bc10d66b63869c4d8f77a06fa2a2f Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Wed, 26 Feb 2025 01:10:57 +0530 Subject: [PATCH 4/7] Review suggestions incorporated. --- src/hotspot/share/opto/vectornode.cpp | 16 ++-- src/hotspot/share/opto/vectornode.hpp | 8 +- .../compiler/lib/ir_framework/IRNode.java | 2 +- .../VectorUnsignedMinMaxOperationsTest.java | 92 +++++++++---------- 4 files changed, 56 insertions(+), 62 deletions(-) diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 4c8207c76f569..9b1d2804a2b2b 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -1042,7 +1042,7 @@ Node* VectorNode::try_to_gen_masked_vector(PhaseGVN* gvn, Node* node, const Type } } -bool VectorNode::should_swap_inputs_to_help_global_value_numbering() { +bool VectorNode::is_commutative() { // Predicated vector operations are sensitive to ordering of inputs. // When the mask corresponding to a vector lane is false then // the result of the operation is corresponding lane of its first operand. @@ -1098,7 +1098,7 @@ Node* VectorNode::Ideal(PhaseGVN* phase, bool can_reshape) { } // Sort inputs of commutative non-predicated vector operations to help value numbering. - if (should_swap_inputs_to_help_global_value_numbering()) { + if (is_commutative()) { swap_edges(1, 2); } return nullptr; @@ -2191,7 +2191,7 @@ bool MulVLNode::has_uint_inputs() const { has_vector_elements_fit_uint(in(2)); } -static Node* unsigned_min_max_xform(Node* n) { +static Node* UMinMaxV_Ideal(Node* n, PhaseGVN* phase, bool can_reshape) { int vopc = n->Opcode(); assert(vopc == Op_UMinV || vopc == Op_UMaxV, "Unexpected opcode"); @@ -2213,9 +2213,9 @@ static Node* unsigned_min_max_xform(Node* n) { // UMin (UMax(a, b), UMin(b, a)) => UMin(a, b) // UMax (UMin(a, b), UMax(a, b)) => UMax(a, b) // UMax (UMax(a, b), UMin(b, a)) => UMax(a, b) - if (umin && umax) { + if (umin != nullptr && umax != nullptr) { if ((umin->in(1) == umax->in(1) && umin->in(2) == umax->in(2)) || - (umin->in(2) == umax->in(1) && umin->in(1) == umax->in(2))) { + (umin->in(2) == umax->in(1) && umin->in(1) == umax->in(2))) { if (vopc == Op_UMinV) { return new UMinVNode(umax->in(1), umax->in(2), n->bottom_type()->is_vect()); } else { @@ -2224,11 +2224,11 @@ static Node* unsigned_min_max_xform(Node* n) { } } - return nullptr; + return static_cast(n)->VectorNode::Ideal(phase, can_reshape); } Node* UMinVNode::Ideal(PhaseGVN* phase, bool can_reshape) { - return unsigned_min_max_xform(this); + return UMinMaxV_Ideal(this, phase, can_reshape); } Node* UMinVNode::Identity(PhaseGVN* phase) { @@ -2240,7 +2240,7 @@ Node* UMinVNode::Identity(PhaseGVN* phase) { } Node* UMaxVNode::Ideal(PhaseGVN* phase, bool can_reshape) { - return unsigned_min_max_xform(this); + return UMinMaxV_Ideal(this, phase, can_reshape); } Node* UMaxVNode::Identity(PhaseGVN* phase) { diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index d1fc2dae876d9..86cb99b5b8db4 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -88,7 +88,7 @@ class VectorNode : public TypeNode { static bool is_convert_opcode(int opc); static bool is_minmax_opcode(int opc); - bool should_swap_inputs_to_help_global_value_numbering(); + bool is_commutative(); static bool is_vshift_cnt_opcode(int opc); @@ -619,9 +619,6 @@ class UMinVNode : public VectorNode { virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); virtual Node* Identity(PhaseGVN* phase); virtual int Opcode() const; - virtual uint hash() const { - return (uintptr_t)in(1) + (uintptr_t)in(2) + Opcode(); - } }; @@ -641,9 +638,6 @@ class UMaxVNode : public VectorNode { virtual Node* Ideal(PhaseGVN* phase, bool can_reshape); virtual Node* Identity(PhaseGVN* phase); virtual int Opcode() const; - virtual uint hash() const { - return (uintptr_t)in(1) + (uintptr_t)in(2) + Opcode(); - } }; //------------------------------AbsVINode-------------------------------------- diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java index fb682b37c4318..bcc3c03e9730d 100644 --- a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java +++ b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java @@ -1254,7 +1254,7 @@ public class IRNode { vectorNode(MUL_ADD_VS2VI, "MulAddVS2VI", TYPE_INT); } - public static final String UMIN_VB = VECTOR_PREFIX + "UMIN_VB" + POSTFIX; + public static final String UMIN_VB = VECTOR_PREFIX + "UMIN_VB" + POSTFIX; static { vectorNode(UMIN_VB, "UMinV", TYPE_BYTE); } diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java index c77c321db95a6..f95e4d4e6e496 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java @@ -23,10 +23,9 @@ /** * @test -* @bug 8338201 -* @summary Support new unsigned and saturating vector operators in VectorAPI +* @bug 8342676 +* @summary Unsigned Vector Min / Max transforms * @modules jdk.incubator.vector -* @requires vm.compiler2.enabled * @library /test/lib / * @run driver compiler.vectorapi.VectorUnsignedMinMaxOperationsTest */ @@ -106,14 +105,14 @@ public VectorUnsignedMinMaxOperationsTest() { } @Test - @IR(counts = {IRNode.UMAX_VB, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @IR(counts = {IRNode.UMAX_VB, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @Warmup(value = 10000) public void umax_byte() { for (int i = 0; i < COUNT; i += bspec.length()) { ByteVector.fromArray(bspec, byte_in1, i) - .lanewise(VectorOperators.UMAX, - ByteVector.fromArray(bspec, byte_in2, i)) - .intoArray(byte_out, i); + .lanewise(VectorOperators.UMAX, + ByteVector.fromArray(bspec, byte_in2, i)) + .intoArray(byte_out, i); } } @@ -129,14 +128,14 @@ public void umax_byte_verify() { } @Test - @IR(counts = {IRNode.UMAX_VS, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @IR(counts = {IRNode.UMAX_VS, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @Warmup(value = 10000) public void umax_short() { for (int i = 0; i < COUNT; i += sspec.length()) { ShortVector.fromArray(sspec, short_in1, i) - .lanewise(VectorOperators.UMAX, - ShortVector.fromArray(sspec, short_in2, i)) - .intoArray(short_out, i); + .lanewise(VectorOperators.UMAX, + ShortVector.fromArray(sspec, short_in2, i)) + .intoArray(short_out, i); } } @@ -152,7 +151,7 @@ public void umax_short_verify() { } @Test - @IR(counts = {IRNode.UMAX_VI, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @IR(counts = {IRNode.UMAX_VI, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @Warmup(value = 10000) public void umax_int() { for (int i = 0; i < COUNT; i += ispec.length()) { @@ -175,14 +174,14 @@ public void umax_int_verify() { } @Test - @IR(counts = {IRNode.UMAX_VL, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @IR(counts = {IRNode.UMAX_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @Warmup(value = 10000) public void umax_long() { for (int i = 0; i < COUNT; i += lspec.length()) { LongVector.fromArray(lspec, long_in1, i) - .lanewise(VectorOperators.UMAX, - LongVector.fromArray(lspec, long_in2, i)) - .intoArray(long_out, i); + .lanewise(VectorOperators.UMAX, + LongVector.fromArray(lspec, long_in2, i)) + .intoArray(long_out, i); } } @@ -198,14 +197,14 @@ public void umax_long_verify() { } @Test - @IR(counts = {IRNode.UMIN_VB, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @IR(counts = {IRNode.UMIN_VB, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @Warmup(value = 10000) public void umin_byte() { for (int i = 0; i < COUNT; i += bspec.length()) { ByteVector.fromArray(bspec, byte_in1, i) - .lanewise(VectorOperators.UMIN, - ByteVector.fromArray(bspec, byte_in2, i)) - .intoArray(byte_out, i); + .lanewise(VectorOperators.UMIN, + ByteVector.fromArray(bspec, byte_in2, i)) + .intoArray(byte_out, i); } } @@ -221,14 +220,14 @@ public void umin_byte_verify() { } @Test - @IR(counts = {IRNode.UMIN_VS, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @IR(counts = {IRNode.UMIN_VS, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @Warmup(value = 10000) public void umin_short() { for (int i = 0; i < COUNT; i += sspec.length()) { ShortVector.fromArray(sspec, short_in1, i) - .lanewise(VectorOperators.UMIN, - ShortVector.fromArray(sspec, short_in2, i)) - .intoArray(short_out, i); + .lanewise(VectorOperators.UMIN, + ShortVector.fromArray(sspec, short_in2, i)) + .intoArray(short_out, i); } } @@ -244,7 +243,7 @@ public void umin_short_verify() { } @Test - @IR(counts = {IRNode.UMIN_VI, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @IR(counts = {IRNode.UMIN_VI, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @Warmup(value = 10000) public void umin_int() { for (int i = 0; i < COUNT; i += ispec.length()) { @@ -267,14 +266,14 @@ public void umin_int_verify() { } @Test - @IR(counts = {IRNode.UMIN_VL, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @IR(counts = {IRNode.UMIN_VL, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @Warmup(value = 10000) public void umin_long() { for (int i = 0; i < COUNT; i += lspec.length()) { LongVector.fromArray(lspec, long_in1, i) - .lanewise(VectorOperators.UMIN, - LongVector.fromArray(lspec, long_in2, i)) - .intoArray(long_out, i); + .lanewise(VectorOperators.UMIN, + LongVector.fromArray(lspec, long_in2, i)) + .intoArray(long_out, i); } } @@ -290,7 +289,7 @@ public void umin_long_verify() { } @Test - @IR(counts = {IRNode.UMIN_VI, " 0 "}, applyIf = {"UseAVX", " >0 "}) + @IR(counts = {IRNode.UMIN_VI, " 0 "}, applyIfCPUFeature = {"avx", "true"}) @Warmup(value = 10000) public void umin_ir_transform1() { for (int i = 0; i < COUNT; i += ispec.length()) { @@ -298,7 +297,7 @@ public void umin_ir_transform1() { .lanewise(VectorOperators.UMIN, IntVector.fromArray(ispec, int_in1, i)) .intoArray(int_out, i); - } + } } @Check(test = "umin_ir_transform1", when = CheckAt.COMPILED) @@ -313,7 +312,7 @@ public void umin_ir_transform1_verify() { } @Test - @IR(counts = {IRNode.UMAX_VI, " 0 "}, applyIf = {"UseAVX", " >0 "}) + @IR(counts = {IRNode.UMAX_VI, " 0 "}, applyIfCPUFeature = {"avx", "true"}) @Warmup(value = 10000) public void umax_ir_transform1() { for (int i = 0; i < COUNT; i += ispec.length()) { @@ -336,7 +335,7 @@ public void umax_ir_transform1_verify() { } @Test - @IR(counts = {IRNode.UMAX_VI, " 0 ", IRNode.UMIN_VI, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @IR(counts = {IRNode.UMAX_VI, " 0 ", IRNode.UMIN_VI, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @Warmup(value = 10000) public void umin_max_ir_transform1() { for (int i = 0; i < COUNT; i += ispec.length()) { @@ -363,14 +362,14 @@ public void umin_max_ir_transform1_verify() { } @Test - @IR(counts = {IRNode.UMIN_VI, " 0 ", IRNode.UMAX_VI, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @IR(counts = {IRNode.UMIN_VI, " 0 ", IRNode.UMAX_VI, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @Warmup(value = 10000) public void umin_max_ir_transform2() { for (int i = 0; i < COUNT; i += ispec.length()) { IntVector vec1 = IntVector.fromArray(ispec, int_in1, i); IntVector vec2 = IntVector.fromArray(ispec, int_in2, i); - // UMinV (UMinV vec1, vec2) (UMaxV vec1, vec2) => UMinV vec1 vec2 - vec1.lanewise(VectorOperators.UMIN, vec2) + // UMaxV (UMinV vec2, vec1) (UMaxV vec1, vec2) => UMinV vec1 vec2 + vec2.lanewise(VectorOperators.UMIN, vec1) .lanewise(VectorOperators.UMAX, vec1.lanewise(VectorOperators.UMAX, vec2)) .intoArray(int_out, i); @@ -381,7 +380,7 @@ public void umin_max_ir_transform2() { public void umin_max_ir_transform2_verify() { for (int i = 0; i < COUNT; i++) { int actual = int_out[i]; - int expected = VectorMath.maxUnsigned(VectorMath.minUnsigned(int_in1[i], int_in2[i]), + int expected = VectorMath.maxUnsigned(VectorMath.minUnsigned(int_in2[i], int_in1[i]), VectorMath.maxUnsigned(int_in1[i], int_in2[i])); if (actual != expected) { throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); @@ -390,7 +389,7 @@ public void umin_max_ir_transform2_verify() { } @Test - @IR(counts = {IRNode.UMAX_VI, " 0 ", IRNode.UMIN_VI, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @IR(counts = {IRNode.UMAX_VI, " 0 ", IRNode.UMIN_VI, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @Warmup(value = 10000) public void umin_max_ir_transform3() { for (int i = 0; i < COUNT; i += ispec.length()) { @@ -417,14 +416,14 @@ public void umin_max_ir_transform3_verify() { } @Test - @IR(counts = {IRNode.UMIN_VI, " 0 ", IRNode.UMAX_VI, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @IR(counts = {IRNode.UMIN_VI, " 0 ", IRNode.UMAX_VI, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @Warmup(value = 10000) public void umin_max_ir_transform4() { for (int i = 0; i < COUNT; i += ispec.length()) { IntVector vec1 = IntVector.fromArray(ispec, int_in1, i); IntVector vec2 = IntVector.fromArray(ispec, int_in2, i); - // UMinV (UMaxV vec1, vec2) (UMaxV vec1, vec2) => UMaxV vec1 vec2 - vec1.lanewise(VectorOperators.UMAX, vec2) + // UMinV (UMaxV vec2, vec1) (UMaxV vec1, vec2) => UMaxV vec1 vec2 + vec2.lanewise(VectorOperators.UMAX, vec1) .lanewise(VectorOperators.UMIN, vec1.lanewise(VectorOperators.UMAX, vec2)) .intoArray(int_out, i); @@ -435,7 +434,7 @@ public void umin_max_ir_transform4() { public void umin_max_ir_transform4_verify() { for (int i = 0; i < COUNT; i++) { int actual = int_out[i]; - int expected = VectorMath.minUnsigned(VectorMath.maxUnsigned(int_in1[i], int_in2[i]), + int expected = VectorMath.minUnsigned(VectorMath.maxUnsigned(int_in2[i], int_in1[i]), VectorMath.maxUnsigned(int_in1[i], int_in2[i])); if (actual != expected) { throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); @@ -444,16 +443,16 @@ public void umin_max_ir_transform4_verify() { } @Test - @IR(counts = {IRNode.UMIN_VI, " 0 ", IRNode.UMAX_VI, " >0 "}, applyIf = {"UseAVX", " >0 "}) + @IR(counts = {IRNode.UMIN_VI, " 0 ", IRNode.UMAX_VI, " >0 "}, applyIfCPUFeature = {"avx", "true"}) @Warmup(value = 10000) public void umin_max_ir_transform5() { for (int i = 0; i < COUNT; i += ispec.length()) { IntVector vec1 = IntVector.fromArray(ispec, int_in1, i); IntVector vec2 = IntVector.fromArray(ispec, int_in2, i); - // UMinV (UMinV vec1, vec2) (UMaxV vec1, vec2) => UMinV vec1 vec2 + // UMaxV (UMinV vec1, vec2) (UMaxV vec2, vec1) => UMinV vec1 vec2 vec1.lanewise(VectorOperators.UMIN, vec2) .lanewise(VectorOperators.UMAX, - vec1.lanewise(VectorOperators.UMAX, vec2)) + vec2.lanewise(VectorOperators.UMAX, vec1)) .intoArray(int_out, i); } } @@ -463,10 +462,11 @@ public void umin_max_ir_transform5_verify() { for (int i = 0; i < COUNT; i++) { int actual = int_out[i]; int expected = VectorMath.maxUnsigned(VectorMath.minUnsigned(int_in1[i], int_in2[i]), - VectorMath.maxUnsigned(int_in1[i], int_in2[i])); + VectorMath.maxUnsigned(int_in2[i], int_in1[i])); if (actual != expected) { throw new AssertionError("Result Mismatch : actual (" + actual + ") != expected (" + expected + ")"); } } } } + From eabebc7481bdb9041aec12d02799a2e9192dbfb9 Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Mon, 14 Apr 2025 18:35:01 +0530 Subject: [PATCH 5/7] Review comments resolutions --- src/hotspot/share/opto/vectornode.cpp | 21 +++++--- src/hotspot/share/opto/vectornode.hpp | 2 +- .../VectorCommutativeOperSharingTest.java | 50 +++++++++++++++++++ 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index c3e799b74a639..05ef64af70432 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -1085,7 +1085,7 @@ Node* VectorNode::try_to_gen_masked_vector(PhaseGVN* gvn, Node* node, const Type } } -bool VectorNode::is_commutative() { +bool VectorNode::should_swap_inputs_to_help_global_value_numbering() { // Predicated vector operations are sensitive to ordering of inputs. // When the mask corresponding to a vector lane is false then // the result of the operation is corresponding lane of its first operand. @@ -1141,7 +1141,7 @@ Node* VectorNode::Ideal(PhaseGVN* phase, bool can_reshape) { } // Sort inputs of commutative non-predicated vector operations to help value numbering. - if (is_commutative()) { + if (should_swap_inputs_to_help_global_value_numbering()) { swap_edges(1, 2); } return nullptr; @@ -2249,10 +2249,11 @@ static Node* UMinMaxV_Ideal(Node* n, PhaseGVN* phase, bool can_reshape) { if (lopc == Op_UMinV && ropc == Op_UMaxV) { umin = n->in(1); umax = n->in(2); - } - else if (lopc == Op_UMaxV && ropc == Op_UMinV) { + } else if (lopc == Op_UMaxV && ropc == Op_UMinV) { umin = n->in(2); umax = n->in(1); + } else { + return nullptr; } // UMin (UMin(a, b), UMax(a, b)) => UMin(a, b) @@ -2270,11 +2271,14 @@ static Node* UMinMaxV_Ideal(Node* n, PhaseGVN* phase, bool can_reshape) { } } - return static_cast(n)->VectorNode::Ideal(phase, can_reshape); + return nullptr; } Node* UMinVNode::Ideal(PhaseGVN* phase, bool can_reshape) { - return UMinMaxV_Ideal(this, phase, can_reshape); + Node* progress = UMinMaxV_Ideal(this, phase, can_reshape); + if (progress != nullptr) return progress; + + return VectorNode::Ideal(phase, can_reshape); } Node* UMinVNode::Identity(PhaseGVN* phase) { @@ -2286,7 +2290,10 @@ Node* UMinVNode::Identity(PhaseGVN* phase) { } Node* UMaxVNode::Ideal(PhaseGVN* phase, bool can_reshape) { - return UMinMaxV_Ideal(this, phase, can_reshape); + Node* progress = UMinMaxV_Ideal(this, phase, can_reshape); + if (progress != nullptr) return progress; + + return VectorNode::Ideal(phase, can_reshape); } Node* UMaxVNode::Identity(PhaseGVN* phase) { diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index 6f76a3075b2c8..a6ce2bc740cd9 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -88,7 +88,7 @@ class VectorNode : public TypeNode { static bool is_convert_opcode(int opc); static bool is_minmax_opcode(int opc); - bool is_commutative(); + bool should_swap_inputs_to_help_global_value_numbering(); static bool is_vshift_cnt_opcode(int opc); diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorCommutativeOperSharingTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorCommutativeOperSharingTest.java index c2237000e6fd2..7bc0d9faf133c 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorCommutativeOperSharingTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorCommutativeOperSharingTest.java @@ -573,4 +573,54 @@ public void checkVectorIRSharing19() { Verify.checkEQ(ir1[i], (mask ? ia[i] + ib[i] : ia[i]) + (mask ? ib[i] + ia[i] : ib[i])); } } + + @Test + @IR(counts = {IRNode.UMAX_VI, IRNode.VECTOR_SIZE_ANY, " 1 "}, applyIfCPUFeatureOr = {"avx", "true"}) + public void testVectorIRSharing20(int index) { + IntVector vec1 = IntVector.fromArray(I_SPECIES, ia, index); + IntVector vec2 = IntVector.fromArray(I_SPECIES, ib, index); + // UMax ((UMax vec1 vec2), (UMax vec2 vec1)) + vec1.lanewise(VectorOperators.UMAX, vec2) + .lanewise(VectorOperators.UMAX, vec2.lanewise(VectorOperators.UMAX, vec1)) + .intoArray(ir1, index); + } + + @Run(test = "testVectorIRSharing20") + public void testVectorIRSharingDriver20() { + for (int i = 0; i < I_SPECIES.loopBound(LENGTH); i += I_SPECIES.length()) { + testVectorIRSharing20(i); + } + checkVectorIRSharing20(); + } + + public void checkVectorIRSharing20() { + for (int i = 0; i < I_SPECIES.loopBound(LENGTH); i++) { + Verify.checkEQ(ir1[i], VectorMath.maxUnsigned(ia[i], ib[i])); + } + } + + @Test + @IR(counts = {IRNode.UMIN_VI, IRNode.VECTOR_SIZE_ANY, " 1 "}, applyIfCPUFeatureOr = {"avx", "true"}) + public void testVectorIRSharing21(int index) { + IntVector vec1 = IntVector.fromArray(I_SPECIES, ia, index); + IntVector vec2 = IntVector.fromArray(I_SPECIES, ib, index); + // UMIN ((UMIN vec1 vec2), (UMIN vec2 vec1)) + vec1.lanewise(VectorOperators.UMIN, vec2) + .lanewise(VectorOperators.UMIN, vec2.lanewise(VectorOperators.UMIN, vec1)) + .intoArray(ir1, index); + } + + @Run(test = "testVectorIRSharing21") + public void testVectorIRSharingDriver21() { + for (int i = 0; i < I_SPECIES.loopBound(LENGTH); i += I_SPECIES.length()) { + testVectorIRSharing21(i); + } + checkVectorIRSharing21(); + } + + public void checkVectorIRSharing21() { + for (int i = 0; i < I_SPECIES.loopBound(LENGTH); i++) { + Verify.checkEQ(ir1[i], VectorMath.minUnsigned(ia[i], ib[i])); + } + } } From 6a2cb63576d6ef5d53aa68987f17be6daf28e5ce Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Mon, 14 Apr 2025 18:41:52 +0530 Subject: [PATCH 6/7] Comment refinement --- .../compiler/vectorapi/VectorCommutativeOperSharingTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorCommutativeOperSharingTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorCommutativeOperSharingTest.java index 7bc0d9faf133c..73718193109ab 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorCommutativeOperSharingTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorCommutativeOperSharingTest.java @@ -579,7 +579,7 @@ public void checkVectorIRSharing19() { public void testVectorIRSharing20(int index) { IntVector vec1 = IntVector.fromArray(I_SPECIES, ia, index); IntVector vec2 = IntVector.fromArray(I_SPECIES, ib, index); - // UMax ((UMax vec1 vec2), (UMax vec2 vec1)) + // UMax ((UMax vec1, vec2), (UMax vec2, vec1)) vec1.lanewise(VectorOperators.UMAX, vec2) .lanewise(VectorOperators.UMAX, vec2.lanewise(VectorOperators.UMAX, vec1)) .intoArray(ir1, index); @@ -604,7 +604,7 @@ public void checkVectorIRSharing20() { public void testVectorIRSharing21(int index) { IntVector vec1 = IntVector.fromArray(I_SPECIES, ia, index); IntVector vec2 = IntVector.fromArray(I_SPECIES, ib, index); - // UMIN ((UMIN vec1 vec2), (UMIN vec2 vec1)) + // UMin ((UMin vec1, vec2), (UMin vec2, vec1)) vec1.lanewise(VectorOperators.UMIN, vec2) .lanewise(VectorOperators.UMIN, vec2.lanewise(VectorOperators.UMIN, vec1)) .intoArray(ir1, index); From b184324b022188e073a7b0d154d82a0f3242e378 Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Fri, 25 Apr 2025 14:47:22 +0530 Subject: [PATCH 7/7] Updating comments --- .../vectorapi/VectorUnsignedMinMaxOperationsTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java index f95e4d4e6e496..d94d65c5143f5 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorUnsignedMinMaxOperationsTest.java @@ -368,7 +368,7 @@ public void umin_max_ir_transform2() { for (int i = 0; i < COUNT; i += ispec.length()) { IntVector vec1 = IntVector.fromArray(ispec, int_in1, i); IntVector vec2 = IntVector.fromArray(ispec, int_in2, i); - // UMaxV (UMinV vec2, vec1) (UMaxV vec1, vec2) => UMinV vec1 vec2 + // UMaxV (UMinV vec2, vec1) (UMaxV vec1, vec2) => UMaxV vec1 vec2 vec2.lanewise(VectorOperators.UMIN, vec1) .lanewise(VectorOperators.UMAX, vec1.lanewise(VectorOperators.UMAX, vec2)) @@ -449,7 +449,7 @@ public void umin_max_ir_transform5() { for (int i = 0; i < COUNT; i += ispec.length()) { IntVector vec1 = IntVector.fromArray(ispec, int_in1, i); IntVector vec2 = IntVector.fromArray(ispec, int_in2, i); - // UMaxV (UMinV vec1, vec2) (UMaxV vec2, vec1) => UMinV vec1 vec2 + // UMaxV (UMinV vec1, vec2) (UMaxV vec2, vec1) => UMaxV vec1 vec2 vec1.lanewise(VectorOperators.UMIN, vec2) .lanewise(VectorOperators.UMAX, vec2.lanewise(VectorOperators.UMAX, vec1))