From 57025e65e1f6fac7414c1ca34d4d1ad08a790bbf Mon Sep 17 00:00:00 2001 From: Vladimir Ivanov Date: Fri, 20 Nov 2020 21:17:09 +0000 Subject: [PATCH] 8256073: Improve vector rematerialization support Reviewed-by: thartmann, kvn --- src/hotspot/share/prims/vectorSupport.cpp | 168 +++++++++------------- src/hotspot/share/prims/vectorSupport.hpp | 9 +- 2 files changed, 73 insertions(+), 104 deletions(-) diff --git a/src/hotspot/share/prims/vectorSupport.cpp b/src/hotspot/share/prims/vectorSupport.cpp index 3d2c1fff59e80..1011dd9b982d1 100644 --- a/src/hotspot/share/prims/vectorSupport.cpp +++ b/src/hotspot/share/prims/vectorSupport.cpp @@ -83,126 +83,94 @@ jint VectorSupport::klass2length(InstanceKlass* ik) { return vlen; } -void VectorSupport::init_vector_array(typeArrayOop arr, BasicType elem_bt, int num_elem, address value_addr) { - int elem_size = type2aelembytes(elem_bt); - for (int i = 0; i < num_elem; i++) { +void VectorSupport::init_payload_element(typeArrayOop arr, bool is_mask, BasicType elem_bt, int index, address addr) { + if (is_mask) { + // Masks require special handling: when boxed they are packed and stored in boolean + // arrays, but in scalarized form they have the same size as corresponding vectors. + // For example, Int512Mask is represented in memory as boolean[16], but + // occupies the whole 512-bit vector register when scalarized. + // (In generated code, the conversion is performed by VectorStoreMask.) + // + // TODO: revisit when predicate registers are fully supported. switch (elem_bt) { - case T_BYTE: { - jbyte elem_value = *(jbyte*) (value_addr + i * elem_size); - arr->byte_at_put(i, elem_value); - break; - } - case T_SHORT: { - jshort elem_value = *(jshort*) (value_addr + i * elem_size); - arr->short_at_put(i, elem_value); - break; - } - case T_INT: { - jint elem_value = *(jint*) (value_addr + i * elem_size); - arr->int_at_put(i, elem_value); - break; - } - case T_LONG: { - jlong elem_value = *(jlong*) (value_addr + i * elem_size); - arr->long_at_put(i, elem_value); - break; - } - case T_FLOAT: { - jfloat elem_value = *(jfloat*) (value_addr + i * elem_size); - arr->float_at_put(i, elem_value); - break; - } - case T_DOUBLE: { - jdouble elem_value = *(jdouble*) (value_addr + i * elem_size); - arr->double_at_put(i, elem_value); - break; - } - default: - fatal("unsupported: %s", type2name(elem_bt)); + case T_BYTE: arr->bool_at_put(index, (*(jbyte*)addr) != 0); break; + case T_SHORT: arr->bool_at_put(index, (*(jshort*)addr) != 0); break; + case T_INT: // fall-through + case T_FLOAT: arr->bool_at_put(index, (*(jint*)addr) != 0); break; + case T_LONG: // fall-through + case T_DOUBLE: arr->bool_at_put(index, (*(jlong*)addr) != 0); break; + + default: fatal("unsupported: %s", type2name(elem_bt)); } - } -} - -void VectorSupport::init_mask_array(typeArrayOop arr, BasicType elem_bt, int num_elem, address value_addr) { - int elem_size = type2aelembytes(elem_bt); - - for (int i = 0; i < num_elem; i++) { + } else { switch (elem_bt) { - case T_BYTE: { - jbyte elem_value = *(jbyte*) (value_addr + i * elem_size); - arr->bool_at_put(i, elem_value != 0); - break; - } - case T_SHORT: { - jshort elem_value = *(jshort*) (value_addr + i * elem_size); - arr->bool_at_put(i, elem_value != 0); - break; - } - case T_INT: // fall-through - case T_FLOAT: { - jint elem_value = *(jint*) (value_addr + i * elem_size); - arr->bool_at_put(i, elem_value != 0); - break; - } - case T_LONG: // fall-through - case T_DOUBLE: { - jlong elem_value = *(jlong*) (value_addr + i * elem_size); - arr->bool_at_put(i, elem_value != 0); - break; - } - default: - fatal("unsupported: %s", type2name(elem_bt)); + case T_BYTE: arr-> byte_at_put(index, *(jbyte*)addr); break; + case T_SHORT: arr-> short_at_put(index, *(jshort*)addr); break; + case T_INT: arr-> int_at_put(index, *(jint*)addr); break; + case T_FLOAT: arr-> float_at_put(index, *(jfloat*)addr); break; + case T_LONG: arr-> long_at_put(index, *(jlong*)addr); break; + case T_DOUBLE: arr->double_at_put(index, *(jdouble*)addr); break; + + default: fatal("unsupported: %s", type2name(elem_bt)); } } } -oop VectorSupport::allocate_vector_payload_helper(InstanceKlass* ik, BasicType elem_bt, int num_elem, address value_addr, TRAPS) { - +Handle VectorSupport::allocate_vector_payload_helper(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, Location location, TRAPS) { bool is_mask = is_vector_mask(ik); + int num_elem = klass2length(ik); + BasicType elem_bt = klass2bt(ik); + int elem_size = type2aelembytes(elem_bt); + // On-heap vector values are represented as primitive arrays. TypeArrayKlass* tak = TypeArrayKlass::cast(Universe::typeArrayKlassObj(is_mask ? T_BOOLEAN : elem_bt)); - typeArrayOop arr = tak->allocate(num_elem, CHECK_NULL); // safepoint + typeArrayOop arr = tak->allocate(num_elem, CHECK_NH); // safepoint - if (is_mask) { - init_mask_array(arr, elem_bt, num_elem, value_addr); + if (location.is_register()) { + // Value was in a callee-saved register. + VMReg vreg = VMRegImpl::as_VMReg(location.register_number()); + + for (int i = 0; i < num_elem; i++) { + int vslot = (i * elem_size) / VMRegImpl::stack_slot_size; + int off = (i * elem_size) % VMRegImpl::stack_slot_size; + + address elem_addr = reg_map->location(vreg->next(vslot)) + off; + init_payload_element(arr, is_mask, elem_bt, i, elem_addr); + } + } else { + // Value was directly saved on the stack. + address base_addr = ((address)fr->unextended_sp()) + location.stack_offset(); + for (int i = 0; i < num_elem; i++) { + init_payload_element(arr, is_mask, elem_bt, i, base_addr + i * elem_size); + } + } + return Handle(THREAD, arr); +} + +Handle VectorSupport::allocate_vector_payload(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, ScopeValue* payload, TRAPS) { + if (payload->is_location() && + payload->as_LocationValue()->location().type() == Location::vector) { + // Vector value in an aligned adjacent tuple (1, 2, 4, 8, or 16 slots). + Location location = payload->as_LocationValue()->location(); + return allocate_vector_payload_helper(ik, fr, reg_map, location, THREAD); // safepoint } else { - init_vector_array(arr, elem_bt, num_elem, value_addr); + // Scalar-replaced boxed vector representation. + StackValue* value = StackValue::create_stack_value(fr, reg_map, payload); + return value->get_obj(); } - return arr; } -oop VectorSupport::allocate_vector(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, ObjectValue* ov, TRAPS) { +instanceOop VectorSupport::allocate_vector(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, ObjectValue* ov, TRAPS) { assert(is_vector(ik), "%s not a vector", ik->name()->as_C_string()); assert(ov->field_size() == 1, "%s not a vector", ik->name()->as_C_string()); - // Vector value in an aligned adjacent tuple (1, 2, 4, 8, or 16 slots). - LocationValue* loc_value = ov->field_at(0)->as_LocationValue(); - - BasicType elem_bt = klass2bt(ik); - int num_elem = klass2length(ik); - - Handle vbox = ik->allocate_instance_handle(CHECK_NULL); - - Location loc = loc_value->location(); - - oop payload = NULL; - if (loc.type() == Location::vector) { - address value_addr = loc.is_register() - // Value was in a callee-save register - ? reg_map->location(VMRegImpl::as_VMReg(loc.register_number())) - // Else value was directly saved on the stack. The frame's original stack pointer, - // before any extension by its callee (due to Compiler1 linkage on SPARC), must be used. - : ((address)fr->unextended_sp()) + loc.stack_offset(); - payload = allocate_vector_payload_helper(ik, elem_bt, num_elem, value_addr, CHECK_NULL); // safepoint - } else { - // assert(false, "interesting"); - StackValue* value = StackValue::create_stack_value(fr, reg_map, loc_value); - payload = value->get_obj()(); - } - vector_VectorPayload::set_payload(vbox(), payload); - return vbox(); + ScopeValue* payload_value = ov->field_at(0); + Handle payload_instance = VectorSupport::allocate_vector_payload(ik, fr, reg_map, payload_value, CHECK_NULL); + instanceOop vbox = ik->allocate_instance(CHECK_NULL); + vector_VectorPayload::set_payload(vbox, payload_instance()); + return vbox; } #ifdef COMPILER2 diff --git a/src/hotspot/share/prims/vectorSupport.hpp b/src/hotspot/share/prims/vectorSupport.hpp index 31be129d07b4f..19e885f9dcf29 100644 --- a/src/hotspot/share/prims/vectorSupport.hpp +++ b/src/hotspot/share/prims/vectorSupport.hpp @@ -39,9 +39,10 @@ extern "C" { class VectorSupport : AllStatic { private: - static void init_mask_array(typeArrayOop arr, BasicType elem_bt, int num_elem, address value_addr); - static void init_vector_array(typeArrayOop arr, BasicType elem_bt, int num_elem, address value_addr); - static oop allocate_vector_payload_helper(InstanceKlass* ik, BasicType elem_bt, int num_elem, address value_addr, TRAPS); + static Handle allocate_vector_payload(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, ScopeValue* payload, TRAPS); + static Handle allocate_vector_payload_helper(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, Location location, TRAPS); + + static void init_payload_element(typeArrayOop arr, bool is_mask, BasicType elem_bt, int index, address addr); static BasicType klass2bt(InstanceKlass* ik); static jint klass2length(InstanceKlass* ik); @@ -81,7 +82,7 @@ class VectorSupport : AllStatic { static int vop2ideal(jint vop, BasicType bt); - static oop allocate_vector(InstanceKlass* holder, frame* fr, RegisterMap* reg_map, ObjectValue* sv, TRAPS); + static instanceOop allocate_vector(InstanceKlass* holder, frame* fr, RegisterMap* reg_map, ObjectValue* sv, TRAPS); static bool is_vector(Klass* klass); static bool is_vector_mask(Klass* klass);