8282389: Add new vector operations to count leading and trailing zeros.

Reviewed-by: jbhateja, psandoz

src/hotspot/share/prims/vectorSupport.cpp

@@ -489,6 +489,26 @@ int VectorSupport::vop2ideal(jint id, BasicType bt) {
       }
       break;
     }
+    case VECTOR_OP_TZ_COUNT: {
+     switch (bt) {
+        case T_BYTE:
+        case T_SHORT:
+        case T_INT:   return Op_CountTrailingZerosI;
+        case T_LONG:  return Op_CountTrailingZerosL;
+        default: fatal("TZ_COUNT: %s", type2name(bt));
+      }
+      break;
+    }
+    case VECTOR_OP_LZ_COUNT: {
+     switch (bt) {
+        case T_BYTE:
+        case T_SHORT:
+        case T_INT:   return Op_CountLeadingZerosI;
+        case T_LONG:  return Op_CountLeadingZerosL;
+        default: fatal("LZ_COUNT: %s", type2name(bt));
+      }
+      break;
+    }
     case VECTOR_OP_TAN:
     case VECTOR_OP_TANH:
     case VECTOR_OP_SIN:

src/hotspot/share/prims/vectorSupport.hpp

@@ -94,6 +94,9 @@ class VectorSupport : AllStatic {
     VECTOR_OP_EXPAND = 27,
     VECTOR_OP_MASK_COMPRESS = 28,
 
+    VECTOR_OP_TZ_COUNT = 29,
+    VECTOR_OP_LZ_COUNT = 30,
+
     // Vector Math Library
     VECTOR_OP_TAN   = 101,
     VECTOR_OP_TANH  = 102,

src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java

@@ -82,6 +82,10 @@ public class VectorSupport {
     public static final int VECTOR_OP_EXPAND = 27;
     public static final int VECTOR_OP_MASK_COMPRESS = 28;
 
+    // Leading/Trailing zeros count operations
+    public static final int VECTOR_OP_TZ_COUNT  = 29;
+    public static final int VECTOR_OP_LZ_COUNT  = 30;
+
     // Math routines
     public static final int VECTOR_OP_TAN = 101;
     public static final int VECTOR_OP_TANH = 102;

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -657,7 +657,10 @@ private static UnaryOperation<ByteVector, VectorMask<Byte>> unaryOperations(int
                     v0.uOp(m, (i, a) -> (byte) Math.abs(a));
             case VECTOR_OP_BIT_COUNT: return (v0, m) ->
                     v0.uOp(m, (i, a) -> (byte) bitCount(a));
-
+            case VECTOR_OP_TZ_COUNT: return (v0, m) ->
+                    v0.uOp(m, (i, a) -> (byte) numberOfTrailingZeros(a));
+            case VECTOR_OP_LZ_COUNT: return (v0, m) ->
+                    v0.uOp(m, (i, a) -> (byte) numberOfLeadingZeros(a));
             default: return null;
         }
     }
@@ -1787,6 +1790,12 @@ ByteVector abs() {
     static int bitCount(byte a) {
         return Integer.bitCount((int)a & 0xFF);
     }
+    static int numberOfTrailingZeros(byte a) {
+        return a != 0 ? Integer.numberOfTrailingZeros(a) : 8;
+    }
+    static int numberOfLeadingZeros(byte a) {
+        return a >= 0 ? Integer.numberOfLeadingZeros(a) - 24 : 0;
+    }
 
     // not (~)
     /**

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -633,7 +633,6 @@ private static UnaryOperation<DoubleVector, VectorMask<Double>> unaryOperations(
                     v0.uOp(m, (i, a) -> (double) -a);
             case VECTOR_OP_ABS: return (v0, m) ->
                     v0.uOp(m, (i, a) -> (double) Math.abs(a));
-
             case VECTOR_OP_SIN: return (v0, m) ->
                     v0.uOp(m, (i, a) -> (double) Math.sin(a));
             case VECTOR_OP_COS: return (v0, m) ->

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -633,7 +633,6 @@ private static UnaryOperation<FloatVector, VectorMask<Float>> unaryOperations(in
                     v0.uOp(m, (i, a) -> (float) -a);
             case VECTOR_OP_ABS: return (v0, m) ->
                     v0.uOp(m, (i, a) -> (float) Math.abs(a));
-
             case VECTOR_OP_SIN: return (v0, m) ->
                     v0.uOp(m, (i, a) -> (float) Math.sin(a));
             case VECTOR_OP_COS: return (v0, m) ->

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -657,7 +657,10 @@ private static UnaryOperation<IntVector, VectorMask<Integer>> unaryOperations(in
                     v0.uOp(m, (i, a) -> (int) Math.abs(a));
             case VECTOR_OP_BIT_COUNT: return (v0, m) ->
                     v0.uOp(m, (i, a) -> (int) Integer.bitCount(a));
-
+            case VECTOR_OP_TZ_COUNT: return (v0, m) ->
+                    v0.uOp(m, (i, a) -> (int) Integer.numberOfTrailingZeros(a));
+            case VECTOR_OP_LZ_COUNT: return (v0, m) ->
+                    v0.uOp(m, (i, a) -> (int) Integer.numberOfLeadingZeros(a));
             default: return null;
         }
     }

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -615,7 +615,10 @@ private static UnaryOperation<LongVector, VectorMask<Long>> unaryOperations(int
                     v0.uOp(m, (i, a) -> (long) Math.abs(a));
             case VECTOR_OP_BIT_COUNT: return (v0, m) ->
                     v0.uOp(m, (i, a) -> (long) Long.bitCount(a));
-
+            case VECTOR_OP_TZ_COUNT: return (v0, m) ->
+                    v0.uOp(m, (i, a) -> (long) Long.numberOfTrailingZeros(a));
+            case VECTOR_OP_LZ_COUNT: return (v0, m) ->
+                    v0.uOp(m, (i, a) -> (long) Long.numberOfLeadingZeros(a));
             default: return null;
         }
     }

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -657,7 +657,10 @@ private static UnaryOperation<ShortVector, VectorMask<Short>> unaryOperations(in
                     v0.uOp(m, (i, a) -> (short) Math.abs(a));
             case VECTOR_OP_BIT_COUNT: return (v0, m) ->
                     v0.uOp(m, (i, a) -> (short) bitCount(a));
-
+            case VECTOR_OP_TZ_COUNT: return (v0, m) ->
+                    v0.uOp(m, (i, a) -> (short) numberOfTrailingZeros(a));
+            case VECTOR_OP_LZ_COUNT: return (v0, m) ->
+                    v0.uOp(m, (i, a) -> (short) numberOfLeadingZeros(a));
             default: return null;
         }
     }
@@ -1787,6 +1790,12 @@ ShortVector abs() {
     static int bitCount(short a) {
         return Integer.bitCount((int)a & 0xFFFF);
     }
+    static int numberOfTrailingZeros(short a) {
+        return a != 0 ? Integer.numberOfTrailingZeros(a) : 16;
+    }
+    static int numberOfLeadingZeros(short a) {
+        return a >= 0 ? Integer.numberOfLeadingZeros(a) - 16 : 0;
+    }
 
     // not (~)
     /**

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorOperators.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -452,6 +452,10 @@ static boolean opKind(Operator op, int bit) {
     public static final Unary NEG = unary("NEG", "-a", VectorSupport.VECTOR_OP_NEG, VO_ALL|VO_SPECIAL);
     /** Produce {@code bitCount(a)} */
     public static final Unary BIT_COUNT = unary("BIT_COUNT", "bitCount", VectorSupport.VECTOR_OP_BIT_COUNT, VO_NOFP);
+    /** Produce {@code numberOfTrailingZeros(a)} */
+    public static final Unary TRAILING_ZEROS_COUNT = unary("TRAILING_ZEROS_COUNT", "numberOfTrailingZeros", VectorSupport.VECTOR_OP_TZ_COUNT, VO_NOFP);
+    /** Produce {@code numberOfLeadingZeros(a)} */
+    public static final Unary LEADING_ZEROS_COUNT = unary("LEADING_ZEROS_COUNT", "numberOfLeadingZeros", VectorSupport.VECTOR_OP_LZ_COUNT, VO_NOFP);
 
     /** Produce {@code sin(a)}.  Floating only.
      *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -682,14 +682,22 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
             case VECTOR_OP_ABS: return (v0, m) ->
                     v0.uOp(m, (i, a) -> ($type$) Math.abs(a));
 #if[!FP]
-            case VECTOR_OP_BIT_COUNT: return (v0, m) ->
 #if[intOrLong]
+            case VECTOR_OP_BIT_COUNT: return (v0, m) ->
                     v0.uOp(m, (i, a) -> ($type$) $Boxtype$.bitCount(a));
+            case VECTOR_OP_TZ_COUNT: return (v0, m) ->
+                    v0.uOp(m, (i, a) -> ($type$) $Boxtype$.numberOfTrailingZeros(a));
+            case VECTOR_OP_LZ_COUNT: return (v0, m) ->
+                    v0.uOp(m, (i, a) -> ($type$) $Boxtype$.numberOfLeadingZeros(a));
 #else[intOrLong]
+            case VECTOR_OP_BIT_COUNT: return (v0, m) ->
                     v0.uOp(m, (i, a) -> ($type$) bitCount(a));
+            case VECTOR_OP_TZ_COUNT: return (v0, m) ->
+                    v0.uOp(m, (i, a) -> ($type$) numberOfTrailingZeros(a));
+            case VECTOR_OP_LZ_COUNT: return (v0, m) ->
+                    v0.uOp(m, (i, a) -> ($type$) numberOfLeadingZeros(a));
 #end[intOrLong]
 #end[!FP]
-
 #if[FP]
             case VECTOR_OP_SIN: return (v0, m) ->
                     v0.uOp(m, (i, a) -> ($type$) Math.sin(a));
@@ -2042,6 +2050,28 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
     }
 #end[!intOrLong]
 #end[!FP]
+#if[!FP]
+#if[!intOrLong]
+    static int numberOfTrailingZeros($type$ a) {
+#if[short]
+        return a != 0 ? Integer.numberOfTrailingZeros(a) : 16;
+#else[short]
+        return a != 0 ? Integer.numberOfTrailingZeros(a) : 8;
+#end[short]
+    }
+#end[!intOrLong]
+#end[!FP]
+#if[!FP]
+#if[!intOrLong]
+    static int numberOfLeadingZeros($type$ a) {
+#if[short]
+        return a >= 0 ? Integer.numberOfLeadingZeros(a) - 16 : 0;
+#else[short]
+        return a >= 0 ? Integer.numberOfLeadingZeros(a) - 24 : 0;
+#end[short]
+    }
+#end[!intOrLong]
+#end[!FP]
 
 #if[BITWISE]
     // not (~)

test/jdk/jdk/incubator/vector/Byte128VectorTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1210,6 +1210,14 @@ static byte ROR_scalar(byte a, byte b) {
         return (byte)(((((byte)a) & 0xFF) >>> (b & 7)) | ((((byte)a) & 0xFF) << (8 - (b & 7))));
     }
 
+    static byte TRAILING_ZEROS_COUNT_scalar(byte a) {
+        return (byte) (a != 0 ? Integer.numberOfTrailingZeros(a) : 8);
+    }
+
+    static byte LEADING_ZEROS_COUNT_scalar(byte a) {
+        return (byte) (a >= 0 ? Integer.numberOfLeadingZeros(a) - 24 : 0);
+    }
+
     static boolean eq(byte a, byte b) {
         return a == b;
     }
@@ -5310,6 +5318,92 @@ static void BIT_COUNTMaskedByte128VectorTests(IntFunction<byte[]> fa,
 
 
 
+
+    static byte TRAILING_ZEROS_COUNT(byte a) {
+        return (byte)(TRAILING_ZEROS_COUNT_scalar(a));
+    }
+
+
+
+    @Test(dataProvider = "byteUnaryOpProvider")
+    static void TRAILING_ZEROS_COUNTByte128VectorTests(IntFunction<byte[]> fa) {
+        byte[] a = fa.apply(SPECIES.length());
+        byte[] r = fr.apply(SPECIES.length());
+
+        for (int ic = 0; ic < INVOC_COUNT; ic++) {
+            for (int i = 0; i < a.length; i += SPECIES.length()) {
+                ByteVector av = ByteVector.fromArray(SPECIES, a, i);
+                av.lanewise(VectorOperators.TRAILING_ZEROS_COUNT).intoArray(r, i);
+            }
+        }
+
+        assertArraysEquals(r, a, Byte128VectorTests::TRAILING_ZEROS_COUNT);
+    }
+
+
+
+    @Test(dataProvider = "byteUnaryOpMaskProvider")
+    static void TRAILING_ZEROS_COUNTMaskedByte128VectorTests(IntFunction<byte[]> fa,
+                                                IntFunction<boolean[]> fm) {
+        byte[] a = fa.apply(SPECIES.length());
+        byte[] r = fr.apply(SPECIES.length());
+        boolean[] mask = fm.apply(SPECIES.length());
+        VectorMask<Byte> vmask = VectorMask.fromArray(SPECIES, mask, 0);
+
+        for (int ic = 0; ic < INVOC_COUNT; ic++) {
+            for (int i = 0; i < a.length; i += SPECIES.length()) {
+                ByteVector av = ByteVector.fromArray(SPECIES, a, i);
+                av.lanewise(VectorOperators.TRAILING_ZEROS_COUNT, vmask).intoArray(r, i);
+            }
+        }
+
+        assertArraysEquals(r, a, mask, Byte128VectorTests::TRAILING_ZEROS_COUNT);
+    }
+
+
+
+    static byte LEADING_ZEROS_COUNT(byte a) {
+        return (byte)(LEADING_ZEROS_COUNT_scalar(a));
+    }
+
+
+
+    @Test(dataProvider = "byteUnaryOpProvider")
+    static void LEADING_ZEROS_COUNTByte128VectorTests(IntFunction<byte[]> fa) {
+        byte[] a = fa.apply(SPECIES.length());
+        byte[] r = fr.apply(SPECIES.length());
+
+        for (int ic = 0; ic < INVOC_COUNT; ic++) {
+            for (int i = 0; i < a.length; i += SPECIES.length()) {
+                ByteVector av = ByteVector.fromArray(SPECIES, a, i);
+                av.lanewise(VectorOperators.LEADING_ZEROS_COUNT).intoArray(r, i);
+            }
+        }
+
+        assertArraysEquals(r, a, Byte128VectorTests::LEADING_ZEROS_COUNT);
+    }
+
+
+
+    @Test(dataProvider = "byteUnaryOpMaskProvider")
+    static void LEADING_ZEROS_COUNTMaskedByte128VectorTests(IntFunction<byte[]> fa,
+                                                IntFunction<boolean[]> fm) {
+        byte[] a = fa.apply(SPECIES.length());
+        byte[] r = fr.apply(SPECIES.length());
+        boolean[] mask = fm.apply(SPECIES.length());
+        VectorMask<Byte> vmask = VectorMask.fromArray(SPECIES, mask, 0);
+
+        for (int ic = 0; ic < INVOC_COUNT; ic++) {
+            for (int i = 0; i < a.length; i += SPECIES.length()) {
+                ByteVector av = ByteVector.fromArray(SPECIES, a, i);
+                av.lanewise(VectorOperators.LEADING_ZEROS_COUNT, vmask).intoArray(r, i);
+            }
+        }
+
+        assertArraysEquals(r, a, mask, Byte128VectorTests::LEADING_ZEROS_COUNT);
+    }
+
+
     @Test(dataProvider = "byteCompareOpProvider")
     static void ltByte128VectorTestsBroadcastSmokeTest(IntFunction<byte[]> fa, IntFunction<byte[]> fb) {
         byte[] a = fa.apply(SPECIES.length());