8265907: JVM crashes when matching VectorMaskCmp Node

Co-authored-by: Wang Huang <whuang@openjdk.org>
Co-authored-by: Ai Jiaming <aijiaming1@huawei.com>
Reviewed-by: njian, jbhateja, sviswanathan, dlong, adinn

src/hotspot/cpu/aarch64/aarch64.ad

@@ -2413,6 +2413,11 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType
         return false;
       }
       break;
+     case Op_VectorMaskCmp:
+       if (vlen < 2 || bit_size < 64) {
+         return false;
+      }
+      break;
     default:
       break;
     }

src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

@@ -1498,7 +1498,11 @@ void C2_MacroAssembler::load_vector_mask(XMMRegister dst, XMMRegister src, int v
 
 void C2_MacroAssembler::load_iota_indices(XMMRegister dst, Register scratch, int vlen_in_bytes) {
   ExternalAddress addr(StubRoutines::x86::vector_iota_indices());
-  if (vlen_in_bytes <= 16) {
+  if (vlen_in_bytes == 4) {
+    movdl(dst, addr);
+  } else if (vlen_in_bytes == 8) {
+    movq(dst, addr);
+  } else if (vlen_in_bytes == 16) {
     movdqu(dst, addr, scratch);
   } else if (vlen_in_bytes == 32) {
     vmovdqu(dst, addr, scratch);
@@ -1507,6 +1511,7 @@ void C2_MacroAssembler::load_iota_indices(XMMRegister dst, Register scratch, int
     evmovdqub(dst, k0, addr, false /*merge*/, Assembler::AVX_512bit, scratch);
   }
 }
+
 // Reductions for vectors of bytes, shorts, ints, longs, floats, and doubles.
 
 void C2_MacroAssembler::reduce_operation_128(BasicType typ, int opcode, XMMRegister dst, XMMRegister src) {

src/hotspot/cpu/x86/x86.ad

@@ -1835,6 +1835,11 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType
         return false;
       }
       break;
+    case Op_VectorMaskCmp:
+      if (vlen < 2 || size_in_bits < 32) {
+        return false;
+      }
+      break;
   }
   return true;  // Per default match rules are supported.
 }
@@ -6918,7 +6923,7 @@ instruct evcmpFD(vec dst, vec src1, vec src2, immI8 cond, rRegP scratch, kReg kt
 instruct vcmp(legVec dst, legVec src1, legVec src2, immI8 cond, rRegP scratch) %{
   predicate((UseAVX <= 2 || !VM_Version::supports_avx512vl()) &&
             !is_unsigned_booltest_pred(n->in(2)->get_int()) &&
-            vector_length_in_bytes(n->in(1)->in(1)) >=  8 && // src1
+            vector_length_in_bytes(n->in(1)->in(1)) >=  4 && // src1
             vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
             is_integral_type(vector_element_basic_type(n->in(1)->in(1)))); // src1
   match(Set dst (VectorMaskCmp (Binary src1 src2) cond));

src/hotspot/share/opto/vectorIntrinsics.cpp

@@ -411,9 +411,6 @@ bool LibraryCallKit::inline_vector_shuffle_iota() {
   int num_elem = vlen->get_con();
   BasicType elem_bt = T_BYTE;
 
-  if (num_elem < 4)
-    return false;
-
   if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) {
     return false;
   }

test/hotspot/jtreg/compiler/vectorapi/TestVectorShuffleIota.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2021, Huawei Technologies Co. Ltd. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package compiler.vectorapi;
+
+import jdk.incubator.vector.IntVector;
+import jdk.incubator.vector.VectorSpecies;
+import jdk.incubator.vector.VectorShuffle;
+
+/*
+ * @test
+ * @bug 8265907
+ * @modules jdk.incubator.vector
+ * @run main/othervm compiler.vectorapi.TestVectorShuffleIota
+ */
+
+public class TestVectorShuffleIota {
+    static final VectorSpecies<Integer> SPECIESi = IntVector.SPECIES_128;
+
+    static final int INVOC_COUNT = 50000;
+
+    static int[] ai = {87, 65, 78, 71};
+
+    static void testShuffleI() {
+        IntVector iv = (IntVector) VectorShuffle.iota(SPECIESi, 0, 2, false).toVector();
+        iv.intoArray(ai, 0);
+    }
+
+    public static void main(String[] args) {
+        for (int i = 0; i < INVOC_COUNT; i++) {
+            testShuffleI();
+        }
+        for (int i = 0; i < ai.length; i++) {
+            System.out.print(ai[i] + ", ");
+        }
+        System.out.println();
+    }
+}