Skip to content
Permalink
Browse files
8271515: Integration of JEP 417: Vector API (Third Incubator)
Co-authored-by: Sandhya Viswanathan <sviswanathan@openjdk.org>
Co-authored-by: Jatin Bhateja <jbhateja@openjdk.org>
Co-authored-by: Ningsheng Jian <njian@openjdk.org>
Co-authored-by: Xiaohong Gong <xgong@openjdk.org>
Co-authored-by: Eric Liu <eliu@openjdk.org>
Co-authored-by: Jie Fu <jiefu@openjdk.org>
Co-authored-by: Vladimir Ivanov <vlivanov@openjdk.org>
Co-authored-by: John R Rose <jrose@openjdk.org>
Co-authored-by: Paul Sandoz <psandoz@openjdk.org>
Co-authored-by: Rado Smogura <mail@smogura.eu>
Reviewed-by: kvn, sviswanathan, ngasson
  • Loading branch information
10 people committed Nov 15, 2021
1 parent 9326eb1 commit a59c9b2ac277d6ff6be1700d91ff389f137e61ca
Showing with 19,895 additions and 5,765 deletions.
  1. +67 −7 src/hotspot/cpu/aarch64/aarch64.ad
  2. +2,699 −1,275 src/hotspot/cpu/aarch64/aarch64_sve.ad
  3. +1,594 −1,119 src/hotspot/cpu/aarch64/aarch64_sve_ad.m4
  4. +128 −25 src/hotspot/cpu/aarch64/assembler_aarch64.hpp
  5. +206 −15 src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
  6. +16 −2 src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
  7. +7 −1 src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
  8. +89 −3 src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
  9. +24 −2 src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
  10. +4 −3 src/hotspot/cpu/aarch64/register_aarch64.cpp
  11. +7 −1 src/hotspot/cpu/aarch64/register_aarch64.hpp
  12. +53 −16 src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
  13. +4 −0 src/hotspot/cpu/arm/arm.ad
  14. +4 −0 src/hotspot/cpu/ppc/ppc.ad
  15. +4 −0 src/hotspot/cpu/s390/s390.ad
  16. +1,896 −40 src/hotspot/cpu/x86/assembler_x86.cpp
  17. +163 −8 src/hotspot/cpu/x86/assembler_x86.hpp
  18. +240 −9 src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
  19. +18 −3 src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
  20. +373 −0 src/hotspot/cpu/x86/macroAssembler_x86.cpp
  21. +96 −1 src/hotspot/cpu/x86/macroAssembler_x86.hpp
  22. +1 −0 src/hotspot/cpu/x86/stubGenerator_x86_32.cpp
  23. +1 −0 src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
  24. +1 −0 src/hotspot/cpu/x86/stubRoutines_x86.cpp
  25. +5 −0 src/hotspot/cpu/x86/stubRoutines_x86.hpp
  26. +1 −0 src/hotspot/cpu/x86/vm_version_x86.hpp
  27. +1,318 −180 src/hotspot/cpu/x86/x86.ad
  28. +3 −1 src/hotspot/share/adlc/forms.cpp
  29. +55 −25 src/hotspot/share/adlc/formssel.cpp
  30. +207 −50 src/hotspot/share/classfile/vmIntrinsics.hpp
  31. +2 −0 src/hotspot/share/opto/c2compiler.cpp
  32. +90 −34 src/hotspot/share/opto/cfgnode.cpp
  33. +3 −0 src/hotspot/share/opto/cfgnode.hpp
  34. +39 −12 src/hotspot/share/opto/chaitin.cpp
  35. +4 −3 src/hotspot/share/opto/chaitin.hpp
  36. +7 −0 src/hotspot/share/opto/classes.hpp
  37. +7 −3 src/hotspot/share/opto/compile.cpp
  38. +2 −1 src/hotspot/share/opto/lcm.cpp
  39. +4 −0 src/hotspot/share/opto/library_call.cpp
  40. +7 −5 src/hotspot/share/opto/library_call.hpp
  41. +59 −0 src/hotspot/share/opto/matcher.cpp
  42. +4 −0 src/hotspot/share/opto/matcher.hpp
  43. +1 −1 src/hotspot/share/opto/memnode.cpp
  44. +11 −2 src/hotspot/share/opto/node.hpp
  45. +4 −5 src/hotspot/share/opto/postaloc.cpp
  46. +1 −1 src/hotspot/share/opto/regmask.cpp
  47. +7 −3 src/hotspot/share/opto/type.cpp
  48. +3 −3 src/hotspot/share/opto/type.hpp
  49. +6 −3 src/hotspot/share/opto/vector.cpp
  50. +966 −242 src/hotspot/share/opto/vectorIntrinsics.cpp
  51. +56 −10 src/hotspot/share/opto/vectornode.cpp
  52. +101 −14 src/hotspot/share/opto/vectornode.hpp
  53. +12 −0 src/hotspot/share/prims/vectorSupport.cpp
  54. +3 −2 src/hotspot/share/prims/vectorSupport.hpp
  55. +4 −0 src/hotspot/share/runtime/vmStructs.cpp
  56. +11 −0 src/hotspot/share/utilities/growableArray.hpp
  57. +102 −5 src/java.base/share/classes/jdk/internal/misc/X-ScopedMemoryAccess.java.template
  58. +282 −134 src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java
  59. +40 −23 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractMask.java
  60. +130 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java
  61. +130 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java
  62. +130 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java
  63. +130 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java
  64. +130 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java
  65. +495 −191 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteVector.java
  66. +122 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java
  67. +122 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java
  68. +122 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java
  69. +122 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java
  70. +122 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java
  71. +511 −195 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java
  72. +122 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java
  73. +122 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java
  74. +122 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java
  75. +122 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java
  76. +122 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java
  77. +474 −195 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java
  78. +129 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java
  79. +129 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java
  80. +129 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java
  81. +129 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java
  82. +129 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java
  83. +535 −204 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntVector.java
  84. +129 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java
  85. +129 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java
  86. +129 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java
  87. +129 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java
  88. +129 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java
  89. +564 −201 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongVector.java
  90. +130 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java
  91. +130 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java
  92. +130 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java
  93. +130 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java
  94. +130 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java
  95. +495 −192 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortVector.java
  96. +21 −1 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorMask.java
  97. +737 −262 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template
  98. +163 −30 src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template
  99. +0 −3 src/jdk.incubator.vector/windows/native/libjsvml/globals_vectorApiSupport_windows.S.inc
  100. +44 −0 test/hotspot/gtest/aarch64/aarch64-asmtest.py
  101. +154 −99 test/hotspot/gtest/aarch64/asmtest.out.h
  102. +468 −0 test/hotspot/jtreg/compiler/vectorapi/VectorMaskCastTest.java
  103. +235 −0 test/hotspot/jtreg/compiler/vectorapi/VectorMaskLoadStoreTest.java
  104. +67 −0 test/hotspot/jtreg/compiler/vectorapi/VectorMemoryAlias.java
@@ -2059,7 +2059,7 @@ uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, boo

assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");

if (src_hi != OptoReg::Bad) {
if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
assert((src_lo&1)==0 && src_lo+1==src_hi &&
(dst_lo&1)==0 && dst_lo+1==dst_hi,
"expected aligned-adjacent pairs");
@@ -2074,7 +2074,7 @@ uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, boo
int src_offset = ra_->reg2offset(src_lo);
int dst_offset = ra_->reg2offset(dst_lo);

if (bottom_type()->isa_vect() != NULL) {
if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
uint ireg = ideal_reg();
if (ireg == Op_VecA && cbuf) {
C2_MacroAssembler _masm(cbuf);
@@ -2180,10 +2180,29 @@ uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, boo
} else if (dst_lo_rc == rc_float) { // stack --> fpr load
__ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
is64 ? __ D : __ S, src_offset);
} else if (dst_lo_rc == rc_predicate) {
__ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
} else { // stack --> stack copy
assert(dst_lo_rc == rc_stack, "spill to bad register class");
__ unspill(rscratch1, is64, src_offset);
__ spill(rscratch1, is64, dst_offset);
if (ideal_reg() == Op_RegVectMask) {
__ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
} else {
__ unspill(rscratch1, is64, src_offset);
__ spill(rscratch1, is64, dst_offset);
}
}
break;
case rc_predicate:
if (dst_lo_rc == rc_predicate) {
__ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
} else if (dst_lo_rc == rc_stack) {
__ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
} else {
assert(false, "bad src and dst rc_class combination.");
ShouldNotReachHere();
}
break;
default:
@@ -2204,7 +2223,7 @@ uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, boo
} else {
st->print("%s", Matcher::regName[dst_lo]);
}
if (bottom_type()->isa_vect() != NULL) {
if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
int vsize = 0;
switch (ideal_reg()) {
case Op_VecD:
@@ -2221,6 +2240,10 @@ uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, boo
ShouldNotReachHere();
}
st->print("\t# vector spill size = %d", vsize);
} else if (ideal_reg() == Op_RegVectMask) {
assert(Matcher::supports_scalable_vector(), "bad register type for spill");
int vsize = Matcher::scalable_predicate_reg_slots() * 32;
st->print("\t# predicate spill size = %d", vsize);
} else {
st->print("\t# spill size = %d", is64 ? 64 : 32);
}
@@ -2382,6 +2405,18 @@ const bool Matcher::match_rule_supported(int opcode) {
ret_value = false;
}
break;
case Op_LoadVectorMasked:
case Op_StoreVectorMasked:
case Op_LoadVectorGatherMasked:
case Op_StoreVectorScatterMasked:
case Op_MaskAll:
case Op_AndVMask:
case Op_OrVMask:
case Op_XorVMask:
if (UseSVE == 0) {
ret_value = false;
}
break;
}

return ret_value; // Per default match rules are supported.
@@ -2430,6 +2465,15 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType
return vector_size_supported(bt, vlen);
}

const bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
// Only SVE supports masked operations.
if (UseSVE == 0) {
return false;
}
return match_rule_supported(opcode) &&
masked_op_sve_supported(opcode, vlen, bt);
}

const RegMask* Matcher::predicate_reg_mask(void) {
return &_PR_REG_mask;
}
@@ -2643,10 +2687,14 @@ bool size_fits_all_mem_uses(AddPNode* addp, int shift) {

// Should the matcher clone input 'm' of node 'n'?
bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
mstack.push(m, Visit); // m = ShiftCntV
// ShiftV src (ShiftCntV con)
// StoreVector (VectorStoreMask src)
if (is_vshift_con_pattern(n, m) ||
(UseSVE > 0 && m->Opcode() == Op_VectorStoreMask && n->Opcode() == Op_StoreVector)) {
mstack.push(m, Visit);
return true;
}

return false;
}

@@ -5505,6 +5553,7 @@ operand pReg()
%{
constraint(ALLOC_IN_RC(pr_reg));
match(RegVectMask);
match(pRegGov);
op_cost(0);
format %{ %}
interface(REG_INTER);
@@ -8854,6 +8903,17 @@ instruct castVV(vReg dst)
ins_pipe(pipe_class_empty);
%}

instruct castVVMask(pRegGov dst)
%{
match(Set dst (CastVV dst));

size(0);
format %{ "# castVV of $dst" %}
ins_encode(/* empty encoding */);
ins_cost(0);
ins_pipe(pipe_class_empty);
%}

// ============================================================================
// Atomic operation instructions
//

1 comment on commit a59c9b2

@openjdk-notifier
Copy link

@openjdk-notifier openjdk-notifier bot commented on a59c9b2 Nov 15, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.