Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions src/hotspot/share/classfile/vmIntrinsics.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -978,8 +978,6 @@ class methodHandle;
\
do_intrinsic(_VectorSelectFromTwoVectorOp, jdk_internal_vm_vector_VectorSupport, vector_select_from_op_name, vector_select_from_op_sig, F_S) \
do_signature(vector_select_from_op_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \
"Ljava/lang/Class;" \
"Ljava/lang/Class;" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
Expand Down
90 changes: 49 additions & 41 deletions src/hotspot/share/opto/vectorIntrinsics.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2634,31 +2634,22 @@ bool LibraryCallKit::inline_vector_extract() {


// public static
// <V extends Vector<E1>,
// VI extends Vector<E2>,
// E1,
// E2>
// V selectFromTwoVectorOp(Class<? extends V> vClass, Class<? extends VI> viClass,
// Class<E1> eClass, Class<E2> iClass, int length,
// VI v1, V v2, V v3,
// <V extends Vector<E>,
// E>
// V selectFromTwoVectorOp(Class<? extends V> vClass, Class<E> eClass, int length,
// V v1, V v2, V v3,
// SelectFromTwoVector<V> defaultImpl)
bool LibraryCallKit::inline_vector_select_from_two_vectors() {
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
const TypeInstPtr* index_vector_klass = gvn().type(argument(1))->isa_instptr();
const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
const TypeInstPtr* index_elem_klass = gvn().type(argument(3))->isa_instptr();
const TypeInt* vlen = gvn().type(argument(4))->isa_int();

if (vector_klass == nullptr || index_vector_klass == nullptr || elem_klass == nullptr ||
index_elem_klass == nullptr || vlen == nullptr || vector_klass->const_oop() == nullptr ||
index_vector_klass->const_oop() == nullptr || elem_klass->const_oop() == nullptr ||
index_elem_klass->const_oop() == nullptr || !vlen->is_con()) {
log_if_needed(" ** missing constant: vclass=%s viclass = %s etype=%s itype = %s vlen=%s",
const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
const TypeInt* vlen = gvn().type(argument(2))->isa_int();

if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr || vector_klass->const_oop() == nullptr ||
elem_klass->const_oop() == nullptr ||!vlen->is_con()) {
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(1)->Opcode()],
NodeClassNames[argument(2)->Opcode()],
NodeClassNames[argument(3)->Opcode()],
NodeClassNames[argument(4)->Opcode()]);
NodeClassNames[argument(2)->Opcode()]);
return false; // not enough info for intrinsification
}

Expand All @@ -2667,30 +2658,28 @@ bool LibraryCallKit::inline_vector_select_from_two_vectors() {
return false;
}

if (!is_klass_initialized(index_vector_klass)) {
log_if_needed(" ** klass argument not initialized");
return false;
}

ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
if (!elem_type->is_primitive_type()) {
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
return false; // should be primitive type
}

ciType* index_elem_type = index_elem_klass->const_oop()->as_instance()->java_mirror_type();
if (!index_elem_type->is_primitive_type()) {
log_if_needed(" ** index element not a primitive bt=%d", index_elem_type->basic_type());
return false; // should be primitive type
int num_elem = vlen->get_con();
if (!is_power_of_2(num_elem)) {
log_if_needed(" ** vlen is not power of two=%d", num_elem);
return false;
}

int num_elem = vlen->get_con();
BasicType elem_bt = elem_type->basic_type();
BasicType index_elem_bt = index_elem_type->basic_type();
assert(!is_floating_point_type(index_elem_bt), "floating point index element type");
BasicType index_elem_bt = elem_bt;
if (elem_bt == T_FLOAT) {
index_elem_bt = T_INT;
} else if (elem_bt == T_DOUBLE) {
index_elem_bt = T_LONG;
}

if (!arch_supports_vector(Op_SelectFromTwoVector, num_elem, elem_bt, VecMaskNotUsed)) {
int cast_vopc = VectorCastNode::opcode(-1, index_elem_bt, true);
int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, T_BYTE, VecMaskNotUsed) ||
!arch_supports_vector(Op_AndV, num_elem, T_BYTE, VecMaskNotUsed) ||
!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
Expand All @@ -2704,30 +2693,49 @@ bool LibraryCallKit::inline_vector_select_from_two_vectors() {
}
}

int cast_vopc = VectorCastNode::opcode(-1, elem_bt, true);
if (is_floating_point_type(elem_bt)) {
index_elem_bt = elem_bt == T_FLOAT ? T_INT : T_LONG;
if (!arch_supports_vector(Op_AndV, num_elem, index_elem_bt, VecMaskNotUsed) ||
!arch_supports_vector(cast_vopc, num_elem, index_elem_bt, VecMaskNotUsed) ||
!arch_supports_vector(Op_Replicate, num_elem, index_elem_bt, VecMaskNotUsed)) {
log_if_needed(" ** index wrapping not supported: vlen=%d etype=%s" ,
num_elem, type2name(elem_bt));
return false; // not supported
}
}

ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
ciKlass* index_vbox_klass = index_vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
const TypeInstPtr* index_vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, index_vbox_klass);

Node* opd1 = unbox_vector(argument(5), index_vbox_type, index_elem_bt, num_elem);
Node* opd1 = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
if (opd1 == nullptr) {
log_if_needed(" ** unbox failed v1=%s",
NodeClassNames[argument(5)->Opcode()]);
NodeClassNames[argument(3)->Opcode()]);
return false;
}
Node* opd2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
Node* opd2 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
if (opd2 == nullptr) {
log_if_needed(" ** unbox failed v2=%s",
NodeClassNames[argument(6)->Opcode()]);
NodeClassNames[argument(4)->Opcode()]);
return false;
}
Node* opd3 = unbox_vector(argument(7), vbox_type, elem_bt, num_elem);
Node* opd3 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
if (opd3 == nullptr) {
log_if_needed(" ** unbox failed v3=%s",
NodeClassNames[argument(7)->Opcode()]);
NodeClassNames[argument(5)->Opcode()]);
return false;
}

if (index_elem_bt != elem_bt) {
opd1 = gvn().transform(VectorCastNode::make(cast_vopc, opd1, index_elem_bt, num_elem));
}

int indexRangeMask = 2 * num_elem - 1;
Node* wrap_mask = gvn().makecon(TypeInteger::make(indexRangeMask, indexRangeMask, Type::WidenMin, index_elem_bt != T_LONG ? T_INT : index_elem_bt));
Node* wrap_mask_vec = gvn().transform(VectorNode::scalar2vector(wrap_mask, num_elem, Type::get_const_basic_type(index_elem_bt), false));
opd1 = gvn().transform(VectorNode::make(Op_AndV, opd1, wrap_mask_vec, opd1->bottom_type()->is_vect()));

const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
Node* operation = gvn().transform(VectorNode::make(Op_SelectFromTwoVector, opd1, opd2, opd3, vt));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -383,20 +383,17 @@ VM binaryOp(int oprId,
}
/* ============================================================================ */

public interface SelectFromTwoVector<V extends Vector<?>, VI extends Vector<?>> {
V apply(VI v1, V v2, V v3);
public interface SelectFromTwoVector<V extends Vector<?>> {
V apply(V v1, V v2, V v3);
}

@IntrinsicCandidate
public static
<V extends Vector<E1>,
VI extends Vector<E2>,
E1,
E2>
V selectFromTwoVectorOp(Class<? extends V> vClass, Class<? extends VI> viClass,
Class<E1> eClass, Class<E2> iClass, int length,
VI v1, V v2, V v3,
SelectFromTwoVector<V,VI> defaultImpl) {
<V extends Vector<E>,
E>
V selectFromTwoVectorOp(Class<? extends V> vClass, Class<E> eClass, int length,
V v1, V v2, V v3,
SelectFromTwoVector<V> defaultImpl) {
assert isNonCapturingLambda(defaultImpl) : defaultImpl;
return defaultImpl.apply(v1, v2, v3);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -511,7 +511,7 @@ public Byte128Vector selectFrom(Vector<Byte> v,
public Byte128Vector selectFrom(Vector<Byte> v1,
Vector<Byte> v2) {
return (Byte128Vector)
super.selectFromTemplate(Byte128Vector.class, (Byte128Vector) v1, (Byte128Vector) v2); // specialize
super.selectFromTemplate((Byte128Vector) v1, (Byte128Vector) v2); // specialize
}

@ForceInline
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -511,7 +511,7 @@ public Byte256Vector selectFrom(Vector<Byte> v,
public Byte256Vector selectFrom(Vector<Byte> v1,
Vector<Byte> v2) {
return (Byte256Vector)
super.selectFromTemplate(Byte256Vector.class, (Byte256Vector) v1, (Byte256Vector) v2); // specialize
super.selectFromTemplate((Byte256Vector) v1, (Byte256Vector) v2); // specialize
}

@ForceInline
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -511,7 +511,7 @@ public Byte512Vector selectFrom(Vector<Byte> v,
public Byte512Vector selectFrom(Vector<Byte> v1,
Vector<Byte> v2) {
return (Byte512Vector)
super.selectFromTemplate(Byte512Vector.class, (Byte512Vector) v1, (Byte512Vector) v2); // specialize
super.selectFromTemplate((Byte512Vector) v1, (Byte512Vector) v2); // specialize
}

@ForceInline
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -511,7 +511,7 @@ public Byte64Vector selectFrom(Vector<Byte> v,
public Byte64Vector selectFrom(Vector<Byte> v1,
Vector<Byte> v2) {
return (Byte64Vector)
super.selectFromTemplate(Byte64Vector.class, (Byte64Vector) v1, (Byte64Vector) v2); // specialize
super.selectFromTemplate((Byte64Vector) v1, (Byte64Vector) v2); // specialize
}

@ForceInline
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -511,7 +511,7 @@ public ByteMaxVector selectFrom(Vector<Byte> v,
public ByteMaxVector selectFrom(Vector<Byte> v1,
Vector<Byte> v2) {
return (ByteMaxVector)
super.selectFromTemplate(ByteMaxVector.class, (ByteMaxVector) v1, (ByteMaxVector) v2); // specialize
super.selectFromTemplate((ByteMaxVector) v1, (ByteMaxVector) v2); // specialize
}

@ForceInline
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -536,15 +536,15 @@ static ByteVector compressHelper(Vector<Byte> v, VectorMask<Byte> m) {
return r;
}

static ByteVector selectFromTwoVectorHelper(Vector<Byte> wrappedIndex, Vector<Byte> src1, Vector<Byte> src2) {
int vlen = wrappedIndex.length();
static ByteVector selectFromTwoVectorHelper(Vector<Byte> indexes, Vector<Byte> src1, Vector<Byte> src2) {
int vlen = indexes.length();
byte[] res = new byte[vlen];
byte[] vecPayload1 = ((ByteVector)wrappedIndex).vec();
byte[] vecPayload1 = ((ByteVector)indexes).vec();
byte[] vecPayload2 = ((ByteVector)src1).vec();
byte[] vecPayload3 = ((ByteVector)src2).vec();
for (int i = 0; i < vlen; i++) {
int index = ((int)vecPayload1[i]);
res[i] = index >= vlen ? vecPayload3[index - vlen] : vecPayload2[index];
int wrapped_index = VectorIntrinsics.wrapToRange((int)vecPayload1[i], 2 * vlen);
res[i] = wrapped_index >= vlen ? vecPayload3[wrapped_index - vlen] : vecPayload2[wrapped_index];
}
return ((ByteVector)src1).vectorFactory(res);
}
Expand Down Expand Up @@ -2592,16 +2592,9 @@ final ByteVector selectFromTemplate(ByteVector v,

/*package-private*/
@ForceInline
final ByteVector selectFromTemplate(Class<? extends Vector<Byte>> indexVecClass,
ByteVector v1, ByteVector v2) {
int vlen = length();
assert ((vlen & (vlen -1)) == 0);
int twoVectorLenMask = (vlen << 1) - 1;
ByteVector wrapped_indexes = this.lanewise(VectorOperators.AND, twoVectorLenMask);
return VectorSupport.selectFromTwoVectorOp(getClass(), indexVecClass, byte.class, byte.class,
vlen, wrapped_indexes, v1, v2,
(vec1, vec2, vec3) -> selectFromTwoVectorHelper(vec1, vec2, vec3)
);
final ByteVector selectFromTemplate(ByteVector v1, ByteVector v2) {
return VectorSupport.selectFromTwoVectorOp(getClass(), byte.class, length(), this, v1, v2,
(vec1, vec2, vec3) -> selectFromTwoVectorHelper(vec1, vec2, vec3));
}

/// Ternary operations
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ public Double128Vector selectFrom(Vector<Double> v,
public Double128Vector selectFrom(Vector<Double> v1,
Vector<Double> v2) {
return (Double128Vector)
super.selectFromTemplate(Long128Vector.class, (Double128Vector) v1, (Double128Vector) v2); // specialize
super.selectFromTemplate((Double128Vector) v1, (Double128Vector) v2); // specialize
}

@ForceInline
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ public Double256Vector selectFrom(Vector<Double> v,
public Double256Vector selectFrom(Vector<Double> v1,
Vector<Double> v2) {
return (Double256Vector)
super.selectFromTemplate(Long256Vector.class, (Double256Vector) v1, (Double256Vector) v2); // specialize
super.selectFromTemplate((Double256Vector) v1, (Double256Vector) v2); // specialize
}

@ForceInline
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ public Double512Vector selectFrom(Vector<Double> v,
public Double512Vector selectFrom(Vector<Double> v1,
Vector<Double> v2) {
return (Double512Vector)
super.selectFromTemplate(Long512Vector.class, (Double512Vector) v1, (Double512Vector) v2); // specialize
super.selectFromTemplate((Double512Vector) v1, (Double512Vector) v2); // specialize
}

@ForceInline
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ public Double64Vector selectFrom(Vector<Double> v,
public Double64Vector selectFrom(Vector<Double> v1,
Vector<Double> v2) {
return (Double64Vector)
super.selectFromTemplate(Long64Vector.class, (Double64Vector) v1, (Double64Vector) v2); // specialize
super.selectFromTemplate((Double64Vector) v1, (Double64Vector) v2); // specialize
}

@ForceInline
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ public DoubleMaxVector selectFrom(Vector<Double> v,
public DoubleMaxVector selectFrom(Vector<Double> v1,
Vector<Double> v2) {
return (DoubleMaxVector)
super.selectFromTemplate(LongMaxVector.class, (DoubleMaxVector) v1, (DoubleMaxVector) v2); // specialize
super.selectFromTemplate((DoubleMaxVector) v1, (DoubleMaxVector) v2); // specialize
}

@ForceInline
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -525,15 +525,15 @@ static DoubleVector compressHelper(Vector<Double> v, VectorMask<Double> m) {
return r;
}

static DoubleVector selectFromTwoVectorHelper(Vector<Long> wrappedIndex, Vector<Double> src1, Vector<Double> src2) {
int vlen = wrappedIndex.length();
static DoubleVector selectFromTwoVectorHelper(Vector<Double> indexes, Vector<Double> src1, Vector<Double> src2) {
int vlen = indexes.length();
double[] res = new double[vlen];
long[] vecPayload1 = ((LongVector)wrappedIndex).vec();
double[] vecPayload1 = ((DoubleVector)indexes).vec();
double[] vecPayload2 = ((DoubleVector)src1).vec();
double[] vecPayload3 = ((DoubleVector)src2).vec();
for (int i = 0; i < vlen; i++) {
int index = ((int)vecPayload1[i]);
res[i] = index >= vlen ? vecPayload3[index - vlen] : vecPayload2[index];
int wrapped_index = VectorIntrinsics.wrapToRange((int)vecPayload1[i], 2 * vlen);
res[i] = wrapped_index >= vlen ? vecPayload3[wrapped_index - vlen] : vecPayload2[wrapped_index];
}
return ((DoubleVector)src1).vectorFactory(res);
}
Expand Down Expand Up @@ -2434,17 +2434,9 @@ final DoubleVector selectFromTemplate(DoubleVector v,

/*package-private*/
@ForceInline
final DoubleVector selectFromTemplate(Class<? extends Vector<Long>> indexVecClass,
DoubleVector v1, DoubleVector v2) {
int vlen = length();
assert ((vlen & (vlen -1)) == 0);
int twoVectorLenMask = (vlen << 1) - 1;
Vector<Long> wrapped_indexes = this.convert(VectorOperators.D2L, 0)
.lanewise(VectorOperators.AND, twoVectorLenMask);
return VectorSupport.selectFromTwoVectorOp(getClass(), indexVecClass , double.class, long.class,
vlen, wrapped_indexes, v1, v2,
(vec1, vec2, vec3) -> selectFromTwoVectorHelper(vec1, vec2, vec3)
);
final DoubleVector selectFromTemplate(DoubleVector v1, DoubleVector v2) {
return VectorSupport.selectFromTwoVectorOp(getClass(), double.class, length(), this, v1, v2,
(vec1, vec2, vec3) -> selectFromTwoVectorHelper(vec1, vec2, vec3));
}

/// Ternary operations
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ public Float128Vector selectFrom(Vector<Float> v,
public Float128Vector selectFrom(Vector<Float> v1,
Vector<Float> v2) {
return (Float128Vector)
super.selectFromTemplate(Int128Vector.class, (Float128Vector) v1, (Float128Vector) v2); // specialize
super.selectFromTemplate((Float128Vector) v1, (Float128Vector) v2); // specialize
}

@ForceInline
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ public Float256Vector selectFrom(Vector<Float> v,
public Float256Vector selectFrom(Vector<Float> v1,
Vector<Float> v2) {
return (Float256Vector)
super.selectFromTemplate(Int256Vector.class, (Float256Vector) v1, (Float256Vector) v2); // specialize
super.selectFromTemplate((Float256Vector) v1, (Float256Vector) v2); // specialize
}

@ForceInline
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ public Float512Vector selectFrom(Vector<Float> v,
public Float512Vector selectFrom(Vector<Float> v1,
Vector<Float> v2) {
return (Float512Vector)
super.selectFromTemplate(Int512Vector.class, (Float512Vector) v1, (Float512Vector) v2); // specialize
super.selectFromTemplate((Float512Vector) v1, (Float512Vector) v2); // specialize
}

@ForceInline
Expand Down
Loading