8282432: Optimize masked "test" Vector API with predicate feature

Reviewed-by: psandoz

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -370,6 +370,12 @@ public final Byte128Mask test(Test op) {
         return super.testTemplate(Byte128Mask.class, op);  // specialize
     }
 
+    @Override
+    @ForceInline
+    public final Byte128Mask test(Test op, VectorMask<Byte> m) {
+        return super.testTemplate(Byte128Mask.class, op, (Byte128Mask) m);  // specialize
+    }
+
     // Specialized comparisons
 
     @Override

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -370,6 +370,12 @@ public final Byte256Mask test(Test op) {
         return super.testTemplate(Byte256Mask.class, op);  // specialize
     }
 
+    @Override
+    @ForceInline
+    public final Byte256Mask test(Test op, VectorMask<Byte> m) {
+        return super.testTemplate(Byte256Mask.class, op, (Byte256Mask) m);  // specialize
+    }
+
     // Specialized comparisons
 
     @Override

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -370,6 +370,12 @@ public final Byte512Mask test(Test op) {
         return super.testTemplate(Byte512Mask.class, op);  // specialize
     }
 
+    @Override
+    @ForceInline
+    public final Byte512Mask test(Test op, VectorMask<Byte> m) {
+        return super.testTemplate(Byte512Mask.class, op, (Byte512Mask) m);  // specialize
+    }
+
     // Specialized comparisons
 
     @Override

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -370,6 +370,12 @@ public final Byte64Mask test(Test op) {
         return super.testTemplate(Byte64Mask.class, op);  // specialize
     }
 
+    @Override
+    @ForceInline
+    public final Byte64Mask test(Test op, VectorMask<Byte> m) {
+        return super.testTemplate(Byte64Mask.class, op, (Byte64Mask) m);  // specialize
+    }
+
     // Specialized comparisons
 
     @Override

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -370,6 +370,12 @@ public final ByteMaxMask test(Test op) {
         return super.testTemplate(ByteMaxMask.class, op);  // specialize
     }
 
+    @Override
+    @ForceInline
+    public final ByteMaxMask test(Test op, VectorMask<Byte> m) {
+        return super.testTemplate(ByteMaxMask.class, op, (ByteMaxMask) m);  // specialize
+    }
+
     // Specialized comparisons
 
     @Override

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1855,12 +1855,11 @@ VectorMask<Byte> lt(byte e) {
     M testTemplate(Class<M> maskType, Test op) {
         ByteSpecies vsp = vspecies();
         if (opKind(op, VO_SPECIAL)) {
-            ByteVector bits = this.viewAsIntegralLanes();
             VectorMask<Byte> m;
             if (op == IS_DEFAULT) {
-                m = bits.compare(EQ, (byte) 0);
+                m = compare(EQ, (byte) 0);
             } else if (op == IS_NEGATIVE) {
-                m = bits.compare(LT, (byte) 0);
+                m = compare(LT, (byte) 0);
             }
             else {
                 throw new AssertionError(op);
@@ -1875,11 +1874,31 @@ M testTemplate(Class<M> maskType, Test op) {
      * {@inheritDoc} <!--workaround-->
      */
     @Override
-    @ForceInline
-    public final
+    public abstract
     VectorMask<Byte> test(VectorOperators.Test op,
-                                  VectorMask<Byte> m) {
-        return test(op).and(m);
+                                  VectorMask<Byte> m);
+
+    /*package-private*/
+    @ForceInline
+    final
+    <M extends VectorMask<Byte>>
+    M testTemplate(Class<M> maskType, Test op, M mask) {
+        ByteSpecies vsp = vspecies();
+        mask.check(maskType, this);
+        if (opKind(op, VO_SPECIAL)) {
+            VectorMask<Byte> m = mask;
+            if (op == IS_DEFAULT) {
+                m = compare(EQ, (byte) 0, m);
+            } else if (op == IS_NEGATIVE) {
+                m = compare(LT, (byte) 0, m);
+            }
+            else {
+                throw new AssertionError(op);
+            }
+            return maskType.cast(m);
+        }
+        int opc = opCode(op);
+        throw new AssertionError(op);
     }
 
     /**

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -357,6 +357,12 @@ public final Double128Mask test(Test op) {
         return super.testTemplate(Double128Mask.class, op);  // specialize
     }
 
+    @Override
+    @ForceInline
+    public final Double128Mask test(Test op, VectorMask<Double> m) {
+        return super.testTemplate(Double128Mask.class, op, (Double128Mask) m);  // specialize
+    }
+
     // Specialized comparisons
 
     @Override

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -357,6 +357,12 @@ public final Double256Mask test(Test op) {
         return super.testTemplate(Double256Mask.class, op);  // specialize
     }
 
+    @Override
+    @ForceInline
+    public final Double256Mask test(Test op, VectorMask<Double> m) {
+        return super.testTemplate(Double256Mask.class, op, (Double256Mask) m);  // specialize
+    }
+
     // Specialized comparisons
 
     @Override

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -357,6 +357,12 @@ public final Double512Mask test(Test op) {
         return super.testTemplate(Double512Mask.class, op);  // specialize
     }
 
+    @Override
+    @ForceInline
+    public final Double512Mask test(Test op, VectorMask<Double> m) {
+        return super.testTemplate(Double512Mask.class, op, (Double512Mask) m);  // specialize
+    }
+
     // Specialized comparisons
 
     @Override

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -357,6 +357,12 @@ public final Double64Mask test(Test op) {
         return super.testTemplate(Double64Mask.class, op);  // specialize
     }
 
+    @Override
+    @ForceInline
+    public final Double64Mask test(Test op, VectorMask<Double> m) {
+        return super.testTemplate(Double64Mask.class, op, (Double64Mask) m);  // specialize
+    }
+
     // Specialized comparisons
 
     @Override

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -357,6 +357,12 @@ public final DoubleMaxMask test(Test op) {
         return super.testTemplate(DoubleMaxMask.class, op);  // specialize
     }
 
+    @Override
+    @ForceInline
+    public final DoubleMaxMask test(Test op, VectorMask<Double> m) {
+        return super.testTemplate(DoubleMaxMask.class, op, (DoubleMaxMask) m);  // specialize
+    }
+
     // Specialized comparisons
 
     @Override

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1715,7 +1715,7 @@ else if (op == IS_FINITE ||
             else {
                 throw new AssertionError(op);
             }
-            return maskType.cast(m.cast(this.vspecies()));
+            return maskType.cast(m.cast(vsp));
         }
         int opc = opCode(op);
         throw new AssertionError(op);
@@ -1725,11 +1725,48 @@ else if (op == IS_FINITE ||
      * {@inheritDoc} <!--workaround-->
      */
     @Override
-    @ForceInline
-    public final
+    public abstract
     VectorMask<Double> test(VectorOperators.Test op,
-                                  VectorMask<Double> m) {
-        return test(op).and(m);
+                                  VectorMask<Double> m);
+
+    /*package-private*/
+    @ForceInline
+    final
+    <M extends VectorMask<Double>>
+    M testTemplate(Class<M> maskType, Test op, M mask) {
+        DoubleSpecies vsp = vspecies();
+        mask.check(maskType, this);
+        if (opKind(op, VO_SPECIAL)) {
+            LongVector bits = this.viewAsIntegralLanes();
+            VectorMask<Long> m = mask.cast(LongVector.species(shape()));
+            if (op == IS_DEFAULT) {
+                m = bits.compare(EQ, (long) 0, m);
+            } else if (op == IS_NEGATIVE) {
+                m = bits.compare(LT, (long) 0, m);
+            }
+            else if (op == IS_FINITE ||
+                     op == IS_NAN ||
+                     op == IS_INFINITE) {
+                // first kill the sign:
+                bits = bits.and(Long.MAX_VALUE);
+                // next find the bit pattern for infinity:
+                long infbits = (long) toBits(Double.POSITIVE_INFINITY);
+                // now compare:
+                if (op == IS_FINITE) {
+                    m = bits.compare(LT, infbits, m);
+                } else if (op == IS_NAN) {
+                    m = bits.compare(GT, infbits, m);
+                } else {
+                    m = bits.compare(EQ, infbits, m);
+                }
+            }
+            else {
+                throw new AssertionError(op);
+            }
+            return maskType.cast(m.cast(vsp));
+        }
+        int opc = opCode(op);
+        throw new AssertionError(op);
     }
 
     /**

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -357,6 +357,12 @@ public final Float128Mask test(Test op) {
         return super.testTemplate(Float128Mask.class, op);  // specialize
     }
 
+    @Override
+    @ForceInline
+    public final Float128Mask test(Test op, VectorMask<Float> m) {
+        return super.testTemplate(Float128Mask.class, op, (Float128Mask) m);  // specialize
+    }
+
     // Specialized comparisons
 
     @Override

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -357,6 +357,12 @@ public final Float256Mask test(Test op) {
         return super.testTemplate(Float256Mask.class, op);  // specialize
     }
 
+    @Override
+    @ForceInline
+    public final Float256Mask test(Test op, VectorMask<Float> m) {
+        return super.testTemplate(Float256Mask.class, op, (Float256Mask) m);  // specialize
+    }
+
     // Specialized comparisons
 
     @Override

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -357,6 +357,12 @@ public final Float512Mask test(Test op) {
         return super.testTemplate(Float512Mask.class, op);  // specialize
     }
 
+    @Override
+    @ForceInline
+    public final Float512Mask test(Test op, VectorMask<Float> m) {
+        return super.testTemplate(Float512Mask.class, op, (Float512Mask) m);  // specialize
+    }
+
     // Specialized comparisons
 
     @Override