Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

8262355: Support for AVX-512 opmask register allocation. #2768

Closed
wants to merge 23 commits into from
Closed
Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
9e1c3e0
8262355: Support for AVX-512 opmask register allocation.
Feb 28, 2021
69003aa
8262355: Fix for AARCH64 build failure.
Feb 28, 2021
ffacbc6
8262355: Creating a new ideal type TypeVectMask for mask generating n…
Mar 4, 2021
4eaa3d8
8262355: Some synthetic changes for cleanup.
Mar 4, 2021
4fadca5
8262355 : Review comments resolution and deopt handling for mask gene…
Mar 10, 2021
f82741e
8262355: Fix for hs-minimal and windows build failures.
Mar 10, 2021
72f02a9
8262355: Fix for windows build failure.
Mar 10, 2021
a46b2bf
8262355: Removing object re-materialization handling for mask-generat…
Mar 11, 2021
df16ac0
8262355: Review comments resolution.
Mar 16, 2021
fcf2cce
8262355: Fix build failure
Mar 16, 2021
847fdce
8262355: Review comments resolutions.
Mar 17, 2021
f1748bf
Merge branch 'master' of http://github.com/openjdk/jdk into JDK-8262355
Mar 17, 2021
29e889e
Merge branch 'master' of http://github.com/openjdk/jdk into JDK-8262355
Mar 18, 2021
7751342
8262355: Review comments resolution
Mar 18, 2021
661fbda
8262355: Extending Type::isa_vect and Type::is_vect routines to TypeV…
Mar 21, 2021
8a05fbb
8262355: Review comments resolution.
Mar 24, 2021
837428d
8262355: Adding missed safety check.
Mar 25, 2021
13e791a
8262355: Updating copywriter for edited files.
Mar 28, 2021
eeea2d7
Merge branch 'master' of http://github.com/openjdk/jdk into JDK-8262355
Mar 29, 2021
5aa0730
8262355: Review comments resolutions.
Apr 1, 2021
366641a
8262355: Fix AARCH64 build issue
Apr 1, 2021
d6bec3d
Merge http://github.com/openjdk/jdk into JDK-8262355
Apr 1, 2021
b9810d2
8262355: Rebasing patch, 32bit clean-up.
Apr 2, 2021
File filter
Filter file types
Jump to
Jump to file
Failed to load files.

Always

Just for now

@@ -2438,16 +2438,9 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType
const bool Matcher::has_predicated_vectors(void) {
return UseSVE > 0;
}
const bool Matcher::is_predicate_operand(int opcode) {
return false;
}
const RegMask* Matcher::predicate_reg_mask(void) {
return NULL;
}
const Type* Matcher::predicate_reg_type() {
assert(has_predicated_vectors(), "");
return Type::BOTTOM;
}

bool Matcher::supports_vector_variable_shifts(void) {
return true;
@@ -992,16 +992,9 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType
const bool Matcher::has_predicated_vectors(void) {
return false;
}
const bool Matcher::is_predicate_operand(int opcode) {
return false;
}
const RegMask* Matcher::predicate_reg_mask(void) {
return NULL;
}
const Type* Matcher::predicate_reg_type() {
assert(has_predicated_vectors(), "");
return Type::BOTTOM;
}

bool Matcher::supports_vector_variable_shifts(void) {
return VM_Version::has_simd();
@@ -2159,16 +2159,9 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType
const bool Matcher::has_predicated_vectors(void) {
return false;
}
const bool Matcher::is_predicate_operand(int opcode) {
return false;
}
const RegMask* Matcher::predicate_reg_mask(void) {
return NULL;
}
const Type* Matcher::predicate_reg_type() {
assert(has_predicated_vectors(), "");
return Type::BOTTOM;
}

bool Matcher::supports_vector_variable_shifts(void) {
return false; // not supported
@@ -1545,16 +1545,9 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType
const bool Matcher::has_predicated_vectors(void) {
return false;
}
const bool Matcher::is_predicate_operand(int opcode) {
return false;
}
const RegMask* Matcher::predicate_reg_mask(void) {
return NULL;
}
const Type* Matcher::predicate_reg_type() {
assert(has_predicated_vectors(), "");
return Type::BOTTOM;
}

bool Matcher::supports_vector_variable_shifts(void) {
return false; // not supported
@@ -31,8 +31,8 @@
Assembler::AvxVectorLen vector_length_encoding(int vlen_in_bytes);

// special instructions for EVEX
void setvectmask(Register dst, Register src, KRegister mask = k1);
void restorevectmask(KRegister mask = k1);
void setvectmask(Register dst, Register src, KRegister mask);
void restorevectmask(KRegister mask);

// Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file.
// See full desription in macroAssembler_x86.cpp.
@@ -289,12 +289,12 @@ OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_
}

void RegisterSaver::restore_live_registers(MacroAssembler* masm, bool restore_vectors) {
int opmask_state_bytes = 0;
int additional_frame_bytes = 0;
int num_xmm_regs = XMMRegisterImpl::number_of_registers;
int ymm_bytes = num_xmm_regs * 16;
int zmm_bytes = num_xmm_regs * 32;
int opmask_state_bytes = KRegisterImpl::number_of_registers * 8;
// Recover XMM & FPU state
int additional_frame_bytes = 0;
#ifdef COMPILER2
if (restore_vectors) {
assert(UseAVX > 0, "Vectors larger than 16 byte long are supported only with AVX");
@@ -307,6 +307,7 @@ void RegisterSaver::restore_live_registers(MacroAssembler* masm, bool restore_ve
}
}
if (UseAVX > 2) {
opmask_state_bytes = KRegisterImpl::number_of_registers * 8;
additional_frame_bytes += opmask_state_bytes;
}
#else
@@ -346,9 +347,11 @@ void RegisterSaver::restore_live_registers(MacroAssembler* masm, bool restore_ve
for (int n = 0; n < num_xmm_regs; n++) {
__ vinsertf64x4_high(as_XMMRegister(n), Address(rsp, n*32+off));
}
#ifdef COMPILER2
for (int n = 0; n < KRegisterImpl::number_of_registers; n++) {
__ kmovql(as_KRegister(n), Address(rsp, n*8));
}
#endif
}
__ addptr(rsp, additional_frame_bytes);
} else {
@@ -1898,30 +1898,6 @@ const RegMask* Matcher::predicate_reg_mask(void) {
return &_OPMASK_REG_mask;
}

const bool Matcher::is_predicate_operand(int opcode) {
if (!has_predicated_vectors()) {
return false;
}
switch(opcode) {
case KREG:
case KREG_K0:
case KREG_K1:
case KREG_K2:
case KREG_K3:
case KREG_K4:
case KREG_K5:
case KREG_K6:
case KREG_K7:
return true;
default:
return false;
}
}

const Type* Matcher::predicate_reg_type() {
return TypeLong::LONG;
}

const int Matcher::float_pressure(int default_pressure_threshold) {
int float_pressure_threshold = default_pressure_threshold;
#ifdef _LP64
@@ -1056,7 +1056,7 @@ uint MachSpillCopyNode::implementation( CodeBuffer *cbuf, PhaseRegAlloc *ra_, bo
if( src_first == dst_first && src_second == dst_second )
return size; // Self copy, no move

if (bottom_type()->isa_vect() != NULL) {
if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
uint ireg = ideal_reg();
assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
assert((src_first_rc != rc_float && dst_first_rc != rc_float), "sanity");
@@ -1148,7 +1148,7 @@ uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
// Self copy, no move
return 0;
}
if (bottom_type()->isa_vect() != NULL) {
if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
uint ireg = ideal_reg();
assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
@@ -946,7 +946,7 @@ const char *ArchDesc::getIdealType(const char *idealOp) {
}

if (strncmp(idealOp, "RegVMask", 8) == 0) {
return "Matcher::predicate_reg_type()";
return "TypeVect::VMASK";
}

// !!!!!
@@ -122,67 +122,6 @@ static Node *merge_region(RegionNode *region, PhaseGVN *phase) {
return progress;
}


void PhiNode::add_req(Node *n) {
Node::add_req(n);
if (Matcher::is_mask_generating_node(n)) {
set_has_masked_inputs(true);
}
}

void PhiNode::add_req_batch(Node* n, uint m) {
Node::add_req_batch(n, m);
if (Matcher::is_mask_generating_node(n)) {
set_has_masked_inputs(true);
}
}
void PhiNode::del_req(uint idx) {
Node::del_req(idx);
bool remove_masked_inputs_flags = true;
for(uint i = 0; i < req() ; i++) {
if(Matcher::is_mask_generating_node(in(i))) {
remove_masked_inputs_flags = false;
}
}
if (remove_masked_inputs_flags) {
set_has_masked_inputs(false);
}
}

void PhiNode::del_req_ordered(uint idx) {
Node::del_req_ordered(idx);
bool remove_masked_inputs_flags = true;
for(uint i = 0; i < req() ; i++) {
if(Matcher::is_mask_generating_node(in(i))) {
remove_masked_inputs_flags = false;
}
}
if (remove_masked_inputs_flags) {
set_has_masked_inputs(false);
}
}

void PhiNode::ins_req(uint i, Node *n) {
Node::ins_req(i, n);
if (Matcher::is_mask_generating_node(n)) {
set_has_masked_inputs(true);
}
}

void PhiNode::set_req(uint i, Node *n) {
Node::set_req(i, n);
if (Matcher::is_mask_generating_node(n)) {
set_has_masked_inputs(true);
}
}

void PhiNode::init_req(uint i, Node *n) {
Node::init_req(i, n);
if (Matcher::is_mask_generating_node(n)) {
set_has_masked_inputs(true);
}
}

//--------------------------------has_phi--------------------------------------
// Helper function: Return any PhiNode that uses this region or NULL
PhiNode* RegionNode::has_phi() const {
@@ -2517,31 +2456,12 @@ const RegMask &PhiNode::in_RegMask(uint i) const {

const RegMask &PhiNode::out_RegMask() const {
uint ideal_reg = _type->ideal_reg();
if (_has_masked_inputs) {
ideal_reg = Op_RegVMask;
}
assert( ideal_reg != Node::NotAMachineReg, "invalid type at Phi" );
if( ideal_reg == 0 ) return RegMask::Empty;
assert(ideal_reg != Op_RegFlags, "flags register is not spillable");
return *(Compile::current()->matcher()->idealreg2spillmask[ideal_reg]);
}

uint PhiNode::ideal_reg() const {
if (_has_masked_inputs) {
return Op_RegVMask;
} else {
return _type->ideal_reg();
}
}

const Type* PhiNode::bottom_type() const {
if (_has_masked_inputs) {
return Matcher::predicate_reg_type();
} else {
return type();
}
}

#ifndef PRODUCT
void PhiNode::related(GrowableArray<Node*> *in_rel, GrowableArray<Node*> *out_rel, bool compact) const {
// For a PhiNode, the set of related nodes includes all inputs till level 2,
@@ -134,7 +134,6 @@ class PhiNode : public TypeNode {
// Array elements references have the same alias_idx but different offset.
const int _inst_offset; // Offset of the instance memory slice.
// Size is bigger to hold the _adr_type field.
bool _has_masked_inputs;
virtual uint hash() const; // Check the type
virtual bool cmp( const Node &n ) const;
virtual uint size_of() const { return sizeof(*this); }
@@ -160,8 +159,7 @@ class PhiNode : public TypeNode {
_inst_mem_id(imid),
_inst_id(iid),
_inst_index(iidx),
_inst_offset(ioffs),
_has_masked_inputs(false)
_inst_offset(ioffs)
{
init_class_id(Class_Phi);
init_req(0, r);
@@ -182,8 +180,6 @@ class PhiNode : public TypeNode {

bool is_tripcount(BasicType bt) const;

bool has_masked_inputs() { return _has_masked_inputs;}
void set_has_masked_inputs(bool val = true) { _has_masked_inputs = val;}
// Determine a unique non-trivial input, if any.
// Ignore casts if it helps. Return NULL on failure.
Node* unique_input(PhaseTransform *phase, bool uncast);
@@ -204,15 +200,6 @@ class PhiNode : public TypeNode {
virtual int Opcode() const;
virtual bool pinned() const { return in(0) != 0; }
virtual const TypePtr *adr_type() const { verify_adr_type(true); return _adr_type; }
virtual uint ideal_reg() const;

virtual void add_req(Node *n);
virtual void add_req_batch(Node* n, uint m);
virtual void del_req(uint idx);
virtual void del_req_ordered(uint idx);
virtual void ins_req(uint i, Node *n);
virtual void set_req(uint i, Node *n);
virtual void init_req(uint i, Node *n);

void set_inst_mem_id(int inst_mem_id) { _inst_mem_id = inst_mem_id; }
const int inst_mem_id() const { return _inst_mem_id; }
@@ -227,7 +214,6 @@ class PhiNode : public TypeNode {
inst_offset() == offset &&
type()->higher_equal(tp);
}
virtual const Type *bottom_type() const;
virtual const Type* Value(PhaseGVN* phase) const;
virtual Node* Identity(PhaseGVN* phase);
virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
@@ -895,15 +895,12 @@ void PhaseChaitin::gather_lrg_masks( bool after_aggressive ) {
break;
case Op_RegL: // Check for long or double
case Op_RegD:
#if defined(IA32) || defined(AMD64)
case Op_RegVMask:
#endif
lrg.set_num_regs(2);
// Define platform specific register pressure
#if defined(ARM32)
lrg.set_reg_pressure(2);
#elif defined(IA32)
if( ireg == Op_RegL || ireg == Op_RegVMask ) {
if( ireg == Op_RegL) {
lrg.set_reg_pressure(2);
} else {
lrg.set_reg_pressure(1);
@@ -921,6 +918,10 @@ void PhaseChaitin::gather_lrg_masks( bool after_aggressive ) {
lrg._is_bound = 1;
}
break;
case Op_RegVMask:
lrg.set_num_regs(RegMask::SlotsPerRegVmask);
lrg.set_reg_pressure(1);
break;
case Op_RegF:
case Op_RegI:
case Op_RegN:
@@ -322,8 +322,6 @@ class MachNode : public Node {
const Type *t = _opnds[0]->type();
if (t == TypeInt::CC) {
return Op_RegFlags;
} else if (Matcher::is_mask_generating_oper(ideal_Opcode())) {
return Op_RegVMask;
} else {
return t->ideal_reg();
}
@@ -1091,14 +1089,6 @@ class MachTempNode : public MachNode {
}
virtual uint size_of() const { return sizeof(MachTempNode); }

virtual uint ideal_reg() const {
if (Matcher::is_predicate_operand(_opnd_array[0]->opcode())) {
return Op_RegVMask;
} else {
return MachNode::ideal_reg();
}
}

#ifndef PRODUCT
virtual void format(PhaseRegAlloc *, outputStream *st ) const {}
virtual const char *Name() const { return "MachTemp";}
@@ -233,7 +233,7 @@ void PhaseMacroExpand::generate_partial_inlining_block(Node** ctrl, MergeMemNode
inline_block = generate_guard(ctrl, bol_le, NULL, PROB_FAIR);
stub_block = *ctrl;

Node* mask_gen = new VectorMaskGenNode(length, Matcher::predicate_reg_type(), Type::get_const_basic_type(type));
Node* mask_gen = new VectorMaskGenNode(length, TypeVect::VMASK, Type::get_const_basic_type(type));
transform_later(mask_gen);

unsigned vec_size = lane_count * type2aelembytes(type);
ProTip! Use n and p to navigate between commits in a pull request.