8251046: [lworld] [lw3] C1 should avoid heap allocations in withfield when possible

src/hotspot/share/c1/c1_GraphBuilder.cpp

@@ -951,12 +951,23 @@ void GraphBuilder::load_local(ValueType* type, int index) {
   Value x = state()->local_at(index);
   assert(x != NULL && !x->type()->is_illegal(), "access of illegal local variable");
   push(type, x);
+  if (x->as_NewInlineTypeInstance() != NULL && x->as_NewInlineTypeInstance()->in_larval_state()) {
+    if (x->as_NewInlineTypeInstance()->on_stack_count() == 1) {
+      x->as_NewInlineTypeInstance()->set_not_larva_anymore();
+    } else {
+      x->as_NewInlineTypeInstance()->increment_on_stack_count();
+    }
+  }
 }
 
 
 void GraphBuilder::store_local(ValueType* type, int index) {
   Value x = pop(type);
   store_local(state(), x, index);
+  if (x->as_NewInlineTypeInstance() != NULL) {
+    x->as_NewInlineTypeInstance()->set_local_index(index);
+    x->as_NewInlineTypeInstance()->decrement_on_stack_count();
+  }
 }
 
 
@@ -986,6 +997,10 @@ void GraphBuilder::store_local(ValueStack* state, Value x, int index) {
 
   x->set_local_index(index);
   state->store_local(index, round_fp(x));
+  if (x->as_NewInlineTypeInstance() != NULL) {
+    x->as_NewInlineTypeInstance()->set_local_index(index);
+    x->as_NewInlineTypeInstance()->decrement_on_stack_count();
+  }
 }
 
 
@@ -1085,27 +1100,31 @@ void GraphBuilder::store_indexed(BasicType type) {
 
 }
 
-
 void GraphBuilder::stack_op(Bytecodes::Code code) {
   switch (code) {
     case Bytecodes::_pop:
-      { state()->raw_pop();
+      { Value w = state()->raw_pop();
+        update_larva_stack_count(w);
       }
       break;
     case Bytecodes::_pop2:
-      { state()->raw_pop();
-        state()->raw_pop();
+      { Value w1 = state()->raw_pop();
+        Value w2 = state()->raw_pop();
+        update_larva_stack_count(w1);
+        update_larva_stack_count(w2);
       }
       break;
     case Bytecodes::_dup:
       { Value w = state()->raw_pop();
+        update_larval_state(w);
         state()->raw_push(w);
         state()->raw_push(w);
       }
       break;
     case Bytecodes::_dup_x1:
       { Value w1 = state()->raw_pop();
         Value w2 = state()->raw_pop();
+        update_larval_state(w1);
         state()->raw_push(w1);
         state()->raw_push(w2);
         state()->raw_push(w1);
@@ -1115,6 +1134,17 @@ void GraphBuilder::stack_op(Bytecodes::Code code) {
       { Value w1 = state()->raw_pop();
         Value w2 = state()->raw_pop();
         Value w3 = state()->raw_pop();
+        // special handling for the dup_x2/pop sequence (see JDK-8251046)
+        if (w1 != NULL && w1->as_NewInlineTypeInstance() != NULL) {
+          ciBytecodeStream s(method());
+          s.force_bci(bci());
+          s.next();
+          if (s.cur_bc() != Bytecodes::_pop) {
+            w1->as_NewInlineTypeInstance()->set_not_larva_anymore();
+          }  else {
+            w1->as_NewInlineTypeInstance()->increment_on_stack_count();
+           }
+        }
         state()->raw_push(w1);
         state()->raw_push(w3);
         state()->raw_push(w2);
@@ -1124,6 +1154,8 @@ void GraphBuilder::stack_op(Bytecodes::Code code) {
     case Bytecodes::_dup2:
       { Value w1 = state()->raw_pop();
         Value w2 = state()->raw_pop();
+        update_larval_state(w1);
+        update_larval_state(w2);
         state()->raw_push(w2);
         state()->raw_push(w1);
         state()->raw_push(w2);
@@ -1134,6 +1166,8 @@ void GraphBuilder::stack_op(Bytecodes::Code code) {
       { Value w1 = state()->raw_pop();
         Value w2 = state()->raw_pop();
         Value w3 = state()->raw_pop();
+        update_larval_state(w1);
+        update_larval_state(w2);
         state()->raw_push(w2);
         state()->raw_push(w1);
         state()->raw_push(w3);
@@ -1146,6 +1180,8 @@ void GraphBuilder::stack_op(Bytecodes::Code code) {
         Value w2 = state()->raw_pop();
         Value w3 = state()->raw_pop();
         Value w4 = state()->raw_pop();
+        update_larval_state(w1);
+        update_larval_state(w2);
         state()->raw_push(w2);
         state()->raw_push(w1);
         state()->raw_push(w4);
@@ -1941,65 +1977,35 @@ void GraphBuilder::withfield(int field_index)
   Value val = pop(type);
   Value obj = apop();
 
-  if (!needs_patching && obj->is_optimizable_for_withfield()) {
-    int astore_index;
-    ciBytecodeStream s(method());
-    s.force_bci(bci());
-    s.next();
-    switch (s.cur_bc()) {
-    case Bytecodes::_astore:    astore_index = s.get_index(); break;
-    case Bytecodes::_astore_0:  astore_index = 0; break;
-    case Bytecodes::_astore_1:  astore_index = 1; break;
-    case Bytecodes::_astore_2:  astore_index = 2; break;
-    case Bytecodes::_astore_3:  astore_index = 3; break;
-    default: astore_index = -1;
-    }
-
-    if (astore_index >= 0 && obj == state()->local_at(astore_index)) {
-      // We have a sequence like this, where we load a value object from a local slot,
-      // and overwrite the same local slot with a modified copy of the inline object.
-      //      defaultvalue #1 // class compiler/valhalla/valuetypes/MyValue1
-      //      astore 9
-      //      ...
-      //      iload_0
-      //      aload 9
-      //      swap
-      //      withfield #7 // Field x:I
-      //      astore 9
-      // If this object was created by defaultvalue, and has not escaped, and is not stored
-      // in any other local slots, we can effectively treat the withfield/astore
-      // sequence as a single putfield bytecode.
-      push(objectType, obj);
-      push(type, val);
-      access_field(Bytecodes::_withfield);
-      stream()->next(); // skip the next astore/astore_n bytecode.
-      return;
-    }
-  }
-
-  assert(holder->is_inlinetype(), "must be an inline klass");
+  assert(holder->is_inlinetype(), "must be a value klass");
   // Save the entire state and re-execute on deopt when executing withfield
   state_before->set_should_reexecute(true);
-  NewInlineTypeInstance* new_instance = new NewInlineTypeInstance(holder->as_inline_klass(), state_before, false);
-  _memory->new_instance(new_instance);
-  apush(append_split(new_instance));
+  NewInlineTypeInstance* new_instance;
+  if (obj->as_NewInlineTypeInstance() != NULL && obj->as_NewInlineTypeInstance()->in_larval_state()) {
+    new_instance = obj->as_NewInlineTypeInstance();
+    apush(append_split(new_instance));
+  } else {
+    new_instance = new NewInlineTypeInstance(holder->as_inline_klass(), state_before, false);
+    _memory->new_instance(new_instance);
+    apush(append_split(new_instance));
 
-  for (int i = 0; i < holder->nof_nonstatic_fields(); i++) {
-    ciField* field = holder->nonstatic_field_at(i);
-    int off = field->offset();
+    for (int i = 0; i < holder->nof_nonstatic_fields(); i++) {
+      ciField* field = holder->nonstatic_field_at(i);
+      int off = field->offset();
 
-    if (field->offset() != offset) {
-      if (field->is_flattened()) {
-        assert(field->type()->is_inlinetype(), "Sanity check");
-        assert(field->type()->is_inlinetype(), "Only inline types can be flattened");
-        ciInlineKlass* vk = field->type()->as_inline_klass();
-        copy_inline_content(vk, obj, off, new_instance, vk->first_field_offset(), state_before, needs_patching);
-      } else {
-        // Only load those fields who are not modified
-        LoadField* load = new LoadField(obj, off, field, false, state_before, needs_patching);
-        Value replacement = append(load);
-        StoreField* store = new StoreField(new_instance, off, field, replacement, false, state_before, needs_patching);
-        append(store);
+      if (field->offset() != offset) {
+        if (field->is_flattened()) {
+          assert(field->type()->is_inlinetype(), "Sanity check");
+          assert(field->type()->is_inlinetype(), "Only inline types can be flattened");
+          ciInlineKlass* vk = field->type()->as_inline_klass();
+          copy_inline_content(vk, obj, off, new_instance, vk->first_field_offset(), state_before, needs_patching);
+        } else {
+          // Only load those fields who are not modified
+          LoadField* load = new LoadField(obj, off, field, false, state_before, needs_patching);
+          Value replacement = append(load);
+          StoreField* store = new StoreField(new_instance, off, field, replacement, false, state_before, needs_patching);
+          append(store);
+        }
       }
     }
   }

src/hotspot/share/c1/c1_GraphBuilder.hpp

@@ -366,6 +366,18 @@ class GraphBuilder {
   // JSR 292 support
   bool try_method_handle_inline(ciMethod* callee, bool ignore_return);
 
+  // Inline type support
+  void update_larval_state(Value v) {
+    if (v != NULL && v->as_NewInlineTypeInstance() != NULL) {
+      v->as_NewInlineTypeInstance()->update_larval_state();
+    }
+  }
+  void update_larva_stack_count(Value v) {
+    if (v != NULL && v->as_NewInlineTypeInstance() != NULL) {
+      v->as_NewInlineTypeInstance()->update_stack_count();
+    }
+  }
+
   // helpers
   void inline_bailout(const char* msg);
   BlockBegin* header_block(BlockBegin* entry, BlockBegin::Flag f, ValueStack* state);

src/hotspot/share/c1/c1_Instruction.cpp

@@ -419,6 +419,40 @@ void BlockBegin::state_values_do(ValueVisitor* f) {
 }
 
 
+StoreField::StoreField(Value obj, int offset, ciField* field, Value value, bool is_static,
+                       ValueStack* state_before, bool needs_patching)
+  : AccessField(obj, offset, field, is_static, state_before, needs_patching)
+  , _value(value)
+{
+  set_flag(NeedsWriteBarrierFlag, as_ValueType(field_type())->is_object());
+#ifdef ASSERT
+  AssertValues assert_value;
+  values_do(&assert_value);
+#endif
+  pin();
+  if (value->as_NewInlineTypeInstance() != NULL) {
+    value->as_NewInlineTypeInstance()->set_not_larva_anymore();
+  }
+}
+
+StoreIndexed::StoreIndexed(Value array, Value index, Value length, BasicType elt_type, Value value,
+                           ValueStack* state_before, bool check_boolean, bool mismatched)
+  : AccessIndexed(array, index, length, elt_type, state_before, mismatched)
+  , _value(value), _check_boolean(check_boolean)
+{
+  set_flag(NeedsWriteBarrierFlag, (as_ValueType(elt_type)->is_object()));
+  set_flag(NeedsStoreCheckFlag, (as_ValueType(elt_type)->is_object()));
+#ifdef ASSERT
+  AssertValues assert_value;
+  values_do(&assert_value);
+#endif
+  pin();
+  if (value->as_NewInlineTypeInstance() != NULL) {
+    value->as_NewInlineTypeInstance()->set_not_larva_anymore();
+  }
+}
+
+
 // Implementation of Invoke
 
 
@@ -446,11 +480,18 @@ Invoke::Invoke(Bytecodes::Code code, ValueType* result_type, Value recv, Values*
   _signature = new BasicTypeList(number_of_arguments() + (has_receiver() ? 1 : 0));
   if (has_receiver()) {
     _signature->append(as_BasicType(receiver()->type()));
+    if (receiver()->as_NewInlineTypeInstance() != NULL) {
+      receiver()->as_NewInlineTypeInstance()->set_not_larva_anymore();
+    }
   }
   for (int i = 0; i < number_of_arguments(); i++) {
-    ValueType* t = argument_at(i)->type();
+    Value v = argument_at(i);
+    ValueType* t = v->type();
     BasicType bt = as_BasicType(t);
     _signature->append(bt);
+    if (v->as_NewInlineTypeInstance() != NULL) {
+      v->as_NewInlineTypeInstance()->set_not_larva_anymore();
+    }
   }
 }
 
@@ -963,6 +1004,8 @@ bool BlockBegin::try_merge(ValueStack* new_state) {
         if (new_value != existing_value && (existing_phi == NULL || existing_phi->block() != this)) {
           existing_state->setup_phi_for_stack(this, index);
           TRACE_PHI(tty->print_cr("creating phi-function %c%d for stack %d", existing_state->stack_at(index)->type()->tchar(), existing_state->stack_at(index)->id(), index));
+          if (new_value->as_NewInlineTypeInstance() != NULL) {new_value->as_NewInlineTypeInstance()->set_not_larva_anymore(); }
+          if (existing_value->as_NewInlineTypeInstance() != NULL) {existing_value->as_NewInlineTypeInstance()->set_not_larva_anymore(); }
         }
       }
 
@@ -977,6 +1020,8 @@ bool BlockBegin::try_merge(ValueStack* new_state) {
         } else if (new_value != existing_value && (existing_phi == NULL || existing_phi->block() != this)) {
           existing_state->setup_phi_for_local(this, index);
           TRACE_PHI(tty->print_cr("creating phi-function %c%d for local %d", existing_state->local_at(index)->type()->tchar(), existing_state->local_at(index)->id(), index));
+          if (new_value->as_NewInlineTypeInstance() != NULL) {new_value->as_NewInlineTypeInstance()->set_not_larva_anymore(); }
+          if (existing_value->as_NewInlineTypeInstance() != NULL) {existing_value->as_NewInlineTypeInstance()->set_not_larva_anymore(); }
         }
       }
     }
@@ -1143,3 +1188,4 @@ void RangeCheckPredicate::check_state() {
 void ProfileInvoke::state_values_do(ValueVisitor* f) {
   if (state() != NULL) state()->values_do(f);
 }
+

src/hotspot/share/c1/c1_Instruction.hpp

@@ -889,14 +889,7 @@ LEAF(StoreField, AccessField)
  public:
   // creation
   StoreField(Value obj, int offset, ciField* field, Value value, bool is_static,
-             ValueStack* state_before, bool needs_patching)
-  : AccessField(obj, offset, field, is_static, state_before, needs_patching)
-  , _value(value)
-  {
-    set_flag(NeedsWriteBarrierFlag, as_ValueType(field_type())->is_object());
-    ASSERT_VALUES
-    pin();
-  }
+             ValueStack* state_before, bool needs_patching);
 
   // accessors
   Value value() const                            { return _value; }
@@ -1037,15 +1030,7 @@ LEAF(StoreIndexed, AccessIndexed)
  public:
   // creation
   StoreIndexed(Value array, Value index, Value length, BasicType elt_type, Value value, ValueStack* state_before,
-               bool check_boolean, bool mismatched = false)
-  : AccessIndexed(array, index, length, elt_type, state_before, mismatched)
-  , _value(value), _check_boolean(check_boolean)
-  {
-    set_flag(NeedsWriteBarrierFlag, (as_ValueType(elt_type)->is_object()));
-    set_flag(NeedsStoreCheckFlag, (as_ValueType(elt_type)->is_object()));
-    ASSERT_VALUES
-    pin();
-  }
+               bool check_boolean, bool mismatched = false);
 
   // accessors
   Value value() const                            { return _value; }
@@ -1376,7 +1361,9 @@ LEAF(NewInlineTypeInstance, StateSplit)
   ciInlineKlass* _klass;
   Value _depends_on;      // Link to instance on with withfield was called on
   bool _is_optimizable_for_withfield;
+  bool _in_larval_state;
   int _first_local_index;
+  int _on_stack_count;
 public:
 
   // Default creation, always allocated for now
@@ -1385,7 +1372,9 @@ LEAF(NewInlineTypeInstance, StateSplit)
    , _is_unresolved(is_unresolved)
    , _klass(klass)
    , _is_optimizable_for_withfield(from_default_value)
+   , _in_larval_state(true)
    , _first_local_index(-1)
+   , _on_stack_count(1)
   {
     if (depends_on == NULL) {
       _depends_on = this;
@@ -1420,11 +1409,28 @@ LEAF(NewInlineTypeInstance, StateSplit)
       if (_first_local_index == -1) {
         _first_local_index = index;
       } else {
-        _is_optimizable_for_withfield = false;
+        set_not_larva_anymore();
       }
     }
   }
-  virtual bool is_optimizable_for_withfield() const {  return _is_optimizable_for_withfield; }
+
+  virtual bool in_larval_state() const { return _in_larval_state; }
+  virtual void set_not_larva_anymore() { _in_larval_state = false; }
+
+  virtual int on_stack_count() { return _on_stack_count; }
+  virtual void increment_on_stack_count() { _on_stack_count++; }
+  virtual void decrement_on_stack_count() { _on_stack_count--; }
+
+  void update_larval_state() {
+    set_not_larva_anymore();
+  }
+
+  void update_stack_count() {
+    if (in_larval_state()) {
+      decrement_on_stack_count();
+    }
+  }
+
 };
 
 BASE(NewArray, StateSplit)