8271589: fatal error with variable shift count integer rotate operation.

Reviewed-by: kvn, sviswanathan

Author: Jatin Bhateja

src/hotspot/share/opto/superword.cpp

@@ -2488,7 +2488,6 @@ void SuperWord::output() {
       } else if (VectorNode::is_scalar_rotate(n)) {
         Node* in1 = low_adr->in(1);
         Node* in2 = p->at(0)->in(2);
-        assert(in2->bottom_type()->isa_int(), "Shift must always be an int value");
         // If rotation count is non-constant or greater than 8bit value create a vector.
         if (!in2->is_Con() || -0x80 > in2->get_int() || in2->get_int() >= 0x80) {
           in2 =  vector_opd(p, 2);

src/hotspot/share/opto/vectornode.cpp

@@ -27,6 +27,7 @@
 #include "opto/mulnode.hpp"
 #include "opto/subnode.hpp"
 #include "opto/vectornode.hpp"
+#include "opto/convertnode.hpp"
 #include "utilities/powerOfTwo.hpp"
 #include "utilities/globalDefinitions.hpp"
 
@@ -311,6 +312,14 @@ bool VectorNode::is_vector_rotate_supported(int vopc, uint vlen, BasicType bt) {
     return true;
   }
 
+  // If target does not support variable shift operations then no point
+  // in creating a rotate vector node since it will not be disintegratable.
+  // Adding a pessimistic check to avoid complex pattern mathing which
+  // may not be full proof.
+  if (!Matcher::supports_vector_variable_shifts()) {
+     return false;
+  }
+
   // Validate existence of nodes created in case of rotate degeneration.
   switch (bt) {
     case T_INT:
@@ -1142,31 +1151,61 @@ Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_
   // later swap them in case of left rotation.
   Node* shiftRCnt = NULL;
   Node* shiftLCnt = NULL;
-  if (cnt->is_Con() && cnt->bottom_type()->isa_int()) {
-    // Constant shift case.
-    int shift = cnt->get_int() & shift_mask;
+  const TypeInt* cnt_type = cnt->bottom_type()->isa_int();
+  bool is_binary_vector_op = false;
+  if (cnt_type && cnt_type->is_con()) {
+    // Constant shift.
+    int shift = cnt_type->get_con() & shift_mask;
     shiftRCnt = phase->intcon(shift);
     shiftLCnt = phase->intcon(shift_mask + 1 - shift);
-  } else {
-    // Variable shift case.
-    assert(VectorNode::is_invariant_vector(cnt), "Broadcast expected");
+  } else if (VectorNode::is_invariant_vector(cnt)) {
+    // Scalar variable shift, handle replicates generated by auto vectorizer.
     cnt = cnt->in(1);
     if (bt == T_LONG) {
       // Shift count vector for Rotate vector has long elements too.
-      assert(cnt->Opcode() == Op_ConvI2L, "ConvI2L expected");
-      cnt = cnt->in(1);
+      if (cnt->Opcode() == Op_ConvI2L) {
+         cnt = cnt->in(1);
+      } else {
+         assert(cnt->bottom_type()->isa_long() &&
+                cnt->bottom_type()->is_long()->is_con(), "Long constant expected");
+         cnt = phase->transform(new ConvL2INode(cnt));
+      }
     }
     shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(shift_mask)));
     shiftLCnt = phase->transform(new SubINode(phase->intcon(shift_mask + 1), shiftRCnt));
+  } else {
+    // Vector variable shift.
+    assert(Matcher::supports_vector_variable_shifts(), "");
+    assert(bt == T_INT, "Variable vector case supported for integer type rotation");
+
+    assert(cnt->bottom_type()->isa_vect(), "Unexpected shift");
+    const Type* elem_ty = Type::get_const_basic_type(bt);
+
+    Node* shift_mask_node = phase->intcon(shift_mask);
+    Node* const_one_node = phase->intcon(1);
+
+    int subVopc = VectorNode::opcode(Op_SubI, bt);
+    int addVopc = VectorNode::opcode(Op_AddI, bt);
+
+    Node* vector_mask = phase->transform(VectorNode::scalar2vector(shift_mask_node, vlen, elem_ty));
+    Node* vector_one = phase->transform(VectorNode::scalar2vector(const_one_node, vlen, elem_ty));
+
+    shiftRCnt = cnt;
+    shiftRCnt = phase->transform(VectorNode::make(Op_AndV, shiftRCnt, vector_mask, vt));
+    vector_mask = phase->transform(VectorNode::make(addVopc, vector_one, vector_mask, vt));
+    shiftLCnt = phase->transform(VectorNode::make(subVopc, vector_mask, shiftRCnt, vt));
+    is_binary_vector_op = true;
   }
 
   // Swap the computed left and right shift counts.
   if (is_rotate_left) {
     swap(shiftRCnt,shiftLCnt);
   }
 
-  shiftLCnt = phase->transform(new LShiftCntVNode(shiftLCnt, vt));
-  shiftRCnt = phase->transform(new RShiftCntVNode(shiftRCnt, vt));
+  if (!is_binary_vector_op) {
+    shiftLCnt = phase->transform(new LShiftCntVNode(shiftLCnt, vt));
+    shiftRCnt = phase->transform(new RShiftCntVNode(shiftRCnt, vt));
+  }
 
   return new OrVNode(phase->transform(VectorNode::make(shiftLOpc, src, shiftLCnt, vlen, bt)),
                      phase->transform(VectorNode::make(shiftROpc, src, shiftRCnt, vlen, bt)),

test/hotspot/jtreg/compiler/c2/cr6340864/TestIntVectRotate.java

@@ -37,13 +37,23 @@
 
 public class TestIntVectRotate {
   private static final int ARRLEN = 997;
+  private static final int REPCOUNT = 16;
   private static final int ITERS  = 11000;
   private static final int ADD_INIT = Integer.MAX_VALUE-500;
   private static final int VALUE = 15;
   private static final int SHIFT = 32;
   private static final int SHIFT_LT_IMM8 = -128;
   private static final int SHIFT_GT_IMM8 = 128;
 
+  private static final int [] rolRes = {
+    15, 32, 68, 144, 304, 640, 1344, 2816, 5888, 12288, 25600, 53248, 110592, 229376, 475136, 983040
+  };
+
+  private static final int [] rorRes = {
+    15, 8, 1073741828, 1073741826, 805306369, -1610612736, 1409286144, 738197504, 385875968, 201326592,
+    104857600, 54525952, 28311552, 14680064, 7602176, 3932160
+  };
+
   public static void main(String args[]) {
     System.out.println("Testing Integer Rotate vectors");
     test();
@@ -58,12 +68,16 @@ public static void main(String args[]) {
 
   static int[] a0 = new int[ARRLEN];
   static int[] a1 = new int[ARRLEN];
+  static int[] a2 = new int[ARRLEN];
+  static int[] a3 = new int[ARRLEN];
 
   static void test() {
     // Initialize
     for (int i=0; i<ARRLEN; i++) {
       int val = (int)(ADD_INIT+i);
       a1[i] = val;
+      a2[i] = (i & (REPCOUNT-1)) + VALUE;
+      a3[i] = i & (REPCOUNT-1);
     }
     System.out.println("Warmup");
     for (int i=0; i<ITERS; i++) {
@@ -93,6 +107,9 @@ static void test() {
       test_rolv(a0, a1, -SHIFT);
       test_rorc_on(a0, a1);
       test_rorv(a0, a1, -SHIFT);
+
+      test_rolv_vec(a0, a2, a3);
+      test_rorv_vec(a0, a2, a3);
     }
   }
 
@@ -188,7 +205,14 @@ static int verify() {
     for (int i=0; i<ARRLEN; i++) {
       errn += verify("test_rolv_on: ", i, a0[i], (int)(((int)(ADD_INIT+i)<<(-SHIFT)) | (int)(ADD_INIT+i)>>>SHIFT));
     }
-
+    test_rolv_vec(a0, a2, a3);
+    for (int i=0; i<ARRLEN; i++) {
+      errn += verify("test_rolv_vec: ", i, a0[i], rolRes[i & (REPCOUNT-1)]);
+    }
+    test_rorv_vec(a0, a2, a3);
+    for (int i=0; i<ARRLEN; i++) {
+      errn += verify("test_rorv_vec: ", i, a0[i], rorRes[i & (REPCOUNT-1)]);
+    }
     test_rorc_on(a0, a1);
     for (int i=0; i<ARRLEN; i++) {
       errn += verify("test_rorc_on: ", i, a0[i], (int)(((int)(ADD_INIT+i)>>>(-SHIFT)) | (int)(ADD_INIT+i)<<SHIFT));
@@ -358,6 +382,20 @@ static void times() {
     }
     end = System.currentTimeMillis();
     System.out.println("test_rorv_on: " + (end - start));
+
+    start = System.currentTimeMillis();
+    for (int i=0; i<ITERS; i++) {
+      test_rorv_vec(a0, a2, a3);
+    }
+    end = System.currentTimeMillis();
+    System.out.println("test_rorv_vec: " + (end - start));
+
+    start = System.currentTimeMillis();
+    for (int i=0; i<ITERS; i++) {
+      test_rolv_vec(a0, a2, a3);
+    }
+    end = System.currentTimeMillis();
+    System.out.println("test_rolv_vec: " + (end - start));
   }
 
   static void test_rolc(int[] a0, int[] a1) {
@@ -384,6 +422,19 @@ static void test_rolc_on(int[] a0, int[] a1) {
     }
   }
 
+  static void test_rolv_vec(int[] a0, int[] a1, int [] a2) {
+    for (int i = 0; i < a0.length; i+=1) {
+      a0[i] = (int)(Integer.rotateLeft(a1[i], a2[i]));
+    }
+  }
+
+  static void test_rorv_vec(int[] a0, int[] a1, int [] a2) {
+    for (int i = 0; i < a0.length; i+=1) {
+      a0[i] = (int)(Integer.rotateRight(a1[i], a2[i]));
+    }
+  }
+
+
   static void test_rolv(int[] a0, int[] a1, int shift) {
     for (int i = 0; i < a0.length; i+=1) {
       a0[i] = (int)(Integer.rotateLeft(a1[i], shift));