8261447: MethodInvocationCounters frequently run into overflow

Reviewed-by: thartmann, mdoerr, kvn, iveresov

src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -70,7 +70,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index) {
 #if (!defined(PRODUCT) && defined(COMPILER2))
   if (CountCompiledCalls) {
     __ lea(r16, ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr()));
-    __ incrementw(Address(r16));
+    __ increment(Address(r16));
   }
 #endif
 
@@ -145,6 +145,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
   if (s == NULL) {
     return NULL;
   }
+
   // Count unused bytes in instruction sequences of variable size.
   // We add them to the computed buffer size in order to avoid
   // overflow in subsequently generated stubs.
@@ -159,7 +160,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
 #if (!defined(PRODUCT) && defined(COMPILER2))
   if (CountCompiledCalls) {
     __ lea(r10, ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr()));
-    __ incrementw(Address(r10));
+    __ increment(Address(r10));
   }
 #endif
 

src/hotspot/cpu/ppc/vtableStubs_ppc_64.cpp

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2018 SAP SE. All rights reserved.
+ * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2021 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -73,9 +73,9 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index) {
     slop_delta  = load_const_maxLen - (__ pc() - start_pc);
     slop_bytes += slop_delta;
     assert(slop_delta >= 0, "negative slop(%d) encountered, adjust code size estimate!", slop_delta);
-    __ lwz(R12_scratch2, offs, R11_scratch1);
+    __ ld(R12_scratch2, offs, R11_scratch1);
     __ addi(R12_scratch2, R12_scratch2, 1);
-    __ stw(R12_scratch2, offs, R11_scratch1);
+    __ std(R12_scratch2, offs, R11_scratch1);
   }
 #endif
 
@@ -141,6 +141,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
   if (s == NULL) {
     return NULL;
   }
+
   // Count unused bytes in instruction sequences of variable size.
   // We add them to the computed buffer size in order to avoid
   // overflow in subsequently generated stubs.
@@ -160,9 +161,9 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
     slop_delta  = load_const_maxLen - (__ pc() - start_pc);
     slop_bytes += slop_delta;
     assert(slop_delta >= 0, "negative slop(%d) encountered, adjust code size estimate!", slop_delta);
-    __ lwz(R12_scratch2, offs, R11_scratch1);
+    __ ld(R12_scratch2, offs, R11_scratch1);
     __ addi(R12_scratch2, R12_scratch2, 1);
-    __ stw(R12_scratch2, offs, R11_scratch1);
+    __ std(R12_scratch2, offs, R11_scratch1);
   }
 #endif
 

src/hotspot/cpu/s390/vtableStubs_s390.cpp

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2018 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2021 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -75,7 +75,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index) {
     // Abuse Z_method as scratch register for generic emitter.
     // It is loaded further down anyway before it is first used.
     // No dynamic code size variance here, increment is 1, always.
-    __ add2mem_32(Address(Z_R1_scratch), 1, Z_method);
+    __ add2mem_64(Address(Z_R1_scratch), 1, Z_method);
   }
 #endif
 
@@ -158,6 +158,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
   if (s == NULL) {
     return NULL;
   }
+
   // Count unused bytes in instruction sequences of variable size.
   // We add them to the computed buffer size in order to avoid
   // overflow in subsequently generated stubs.
@@ -179,7 +180,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
     // Abuse Z_method as scratch register for generic emitter.
     // It is loaded further down anyway before it is first used.
     // No dynamic code size variance here, increment is 1, always.
-    __ add2mem_32(Address(Z_R1_scratch), 1, Z_method);
+    __ add2mem_64(Address(Z_R1_scratch), 1, Z_method);
   }
 #endif
 

src/hotspot/cpu/x86/vtableStubs_x86_64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -70,7 +70,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index) {
 
 #if (!defined(PRODUCT) && defined(COMPILER2))
   if (CountCompiledCalls) {
-    __ incrementl(ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr()));
+    __ incrementq(ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr()));
   }
 #endif
 
@@ -148,6 +148,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
   if (s == NULL) {
     return NULL;
   }
+
   // Count unused bytes in instruction sequences of variable size.
   // We add them to the computed buffer size in order to avoid
   // overflow in subsequently generated stubs.
@@ -163,7 +164,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
 
 #if (!defined(PRODUCT) && defined(COMPILER2))
   if (CountCompiledCalls) {
-    __ incrementl(ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr()));
+    __ incrementq(ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr()));
   }
 #endif // PRODUCT
 

src/hotspot/share/oops/method.cpp

@@ -493,6 +493,7 @@ bool Method::was_executed_more_than(int n) {
 }
 
 void Method::print_invocation_count() {
+  //---<  compose+print method return type, klass, name, and signature  >---
   if (is_static()) tty->print("static ");
   if (is_final()) tty->print("final ");
   if (is_synchronized()) tty->print("synchronized ");
@@ -507,12 +508,22 @@ void Method::print_invocation_count() {
   }
   tty->cr();
 
-  tty->print_cr ("  interpreter_invocation_count: %8d ", interpreter_invocation_count());
-  tty->print_cr ("  invocation_counter:           %8d ", invocation_count());
-  tty->print_cr ("  backedge_counter:             %8d ", backedge_count());
+  // Counting based on signed int counters tends to overflow with
+  // longer-running workloads on fast machines. The counters under
+  // consideration here, however, are limited in range by counting
+  // logic. See InvocationCounter:count_limit for example.
+  // No "overflow precautions" need to be implemented here.
+  tty->print_cr ("  interpreter_invocation_count: " INT32_FORMAT_W(11), interpreter_invocation_count());
+  tty->print_cr ("  invocation_counter:           " INT32_FORMAT_W(11), invocation_count());
+  tty->print_cr ("  backedge_counter:             " INT32_FORMAT_W(11), backedge_count());
+
+  if (method_data() != NULL) {
+    tty->print_cr ("  decompile_count:              " UINT32_FORMAT_W(11), method_data()->decompile_count());
+  }
+
 #ifndef PRODUCT
   if (CountCompiledCalls) {
-    tty->print_cr ("  compiled_invocation_count: %8d ", compiled_invocation_count());
+    tty->print_cr ("  compiled_invocation_count:    " INT64_FORMAT_W(11), compiled_invocation_count());
   }
 #endif
 }

src/hotspot/share/oops/method.hpp

@@ -98,7 +98,7 @@ class Method : public Metadata {
   JFR_ONLY(DEFINE_TRACE_FLAG;)
 
 #ifndef PRODUCT
-  int               _compiled_invocation_count;  // Number of nmethod invocations so far (for perf. debugging)
+  int64_t _compiled_invocation_count;
 #endif
   // Entry point for calling both from and to the interpreter.
   address _i2i_entry;           // All-args-on-stack calling convention
@@ -436,11 +436,11 @@ class Method : public Metadata {
   int interpreter_invocation_count()            { return invocation_count();          }
 
 #ifndef PRODUCT
-  int  compiled_invocation_count() const        { return _compiled_invocation_count;  }
-  void set_compiled_invocation_count(int count) { _compiled_invocation_count = count; }
+  int64_t  compiled_invocation_count() const    { return _compiled_invocation_count;}
+  void set_compiled_invocation_count(int count) { _compiled_invocation_count = (int64_t)count; }
 #else
   // for PrintMethodData in a product build
-  int  compiled_invocation_count() const        { return 0;  }
+  int64_t  compiled_invocation_count() const    { return 0; }
 #endif // not PRODUCT
 
   // Clear (non-shared space) pointers which could not be relevant

src/hotspot/share/opto/graphKit.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -2000,9 +2000,9 @@ void GraphKit::increment_counter(address counter_addr) {
 void GraphKit::increment_counter(Node* counter_addr) {
   int adr_type = Compile::AliasIdxRaw;
   Node* ctrl = control();
-  Node* cnt  = make_load(ctrl, counter_addr, TypeInt::INT, T_INT, adr_type, MemNode::unordered);
-  Node* incr = _gvn.transform(new AddINode(cnt, _gvn.intcon(1)));
-  store_to_memory(ctrl, counter_addr, incr, T_INT, adr_type, MemNode::unordered);
+  Node* cnt  = make_load(ctrl, counter_addr, TypeLong::LONG, T_LONG, adr_type, MemNode::unordered);
+  Node* incr = _gvn.transform(new AddLNode(cnt, _gvn.longcon(1)));
+  store_to_memory(ctrl, counter_addr, incr, T_LONG, adr_type, MemNode::unordered);
 }
 
 

src/hotspot/share/runtime/java.cpp

@@ -97,9 +97,11 @@
 GrowableArray<Method*>* collected_profiled_methods;
 
 int compare_methods(Method** a, Method** b) {
-  // %%% there can be 32-bit overflow here
-  return ((*b)->invocation_count() + (*b)->compiled_invocation_count())
-       - ((*a)->invocation_count() + (*a)->compiled_invocation_count());
+  // compiled_invocation_count() returns int64_t, forcing the entire expression
+  // to be evaluated as int64_t. Overflow is not an issue.
+  int64_t diff = (((*b)->invocation_count() + (*b)->compiled_invocation_count())
+                - ((*a)->invocation_count() + (*a)->compiled_invocation_count()));
+  return (diff < 0) ? -1 : (diff > 0) ? 1 : 0;
 }
 
 void collect_profiled_methods(Method* m) {
@@ -151,14 +153,15 @@ void print_method_profiling_data() {
 GrowableArray<Method*>* collected_invoked_methods;
 
 void collect_invoked_methods(Method* m) {
-  if (m->invocation_count() + m->compiled_invocation_count() >= 1 ) {
+  if (m->invocation_count() + m->compiled_invocation_count() >= 1) {
     collected_invoked_methods->push(m);
   }
 }
 
 
-
-
+// Invocation count accumulators should be unsigned long to shift the
+// overflow border. Longer-running workloads tend to create invocation
+// counts which already overflow 32-bit counters for individual methods.
 void print_method_invocation_histogram() {
   ResourceMark rm;
   collected_invoked_methods = new GrowableArray<Method*>(1024);
@@ -169,31 +172,45 @@ void print_method_invocation_histogram() {
   tty->print_cr("Histogram Over Method Invocation Counters (cutoff = " INTX_FORMAT "):", MethodHistogramCutoff);
   tty->cr();
   tty->print_cr("____Count_(I+C)____Method________________________Module_________________");
-  unsigned total = 0, int_total = 0, comp_total = 0, static_total = 0, final_total = 0,
-      synch_total = 0, nativ_total = 0, acces_total = 0;
+  uint64_t total        = 0,
+           int_total    = 0,
+           comp_total   = 0,
+           special_total= 0,
+           static_total = 0,
+           final_total  = 0,
+           synch_total  = 0,
+           native_total = 0,
+           access_total = 0;
   for (int index = 0; index < collected_invoked_methods->length(); index++) {
+    // Counter values returned from getter methods are signed int.
+    // To shift the overflow border by a factor of two, we interpret
+    // them here as unsigned long. A counter can't be negative anyway.
     Method* m = collected_invoked_methods->at(index);
-    int c = m->invocation_count() + m->compiled_invocation_count();
-    if (c >= MethodHistogramCutoff) m->print_invocation_count();
-    int_total  += m->invocation_count();
-    comp_total += m->compiled_invocation_count();
-    if (m->is_final())        final_total  += c;
-    if (m->is_static())       static_total += c;
-    if (m->is_synchronized()) synch_total  += c;
-    if (m->is_native())       nativ_total  += c;
-    if (m->is_accessor())     acces_total  += c;
+    uint64_t iic = (uint64_t)m->invocation_count();
+    uint64_t cic = (uint64_t)m->compiled_invocation_count();
+    if ((iic + cic) >= (uint64_t)MethodHistogramCutoff) m->print_invocation_count();
+    int_total  += iic;
+    comp_total += cic;
+    if (m->is_final())        final_total  += iic + cic;
+    if (m->is_static())       static_total += iic + cic;
+    if (m->is_synchronized()) synch_total  += iic + cic;
+    if (m->is_native())       native_total += iic + cic;
+    if (m->is_accessor())     access_total += iic + cic;
   }
   tty->cr();
   total = int_total + comp_total;
-  tty->print_cr("Invocations summary:");
-  tty->print_cr("\t%9d (%4.1f%%) interpreted",  int_total,    100.0 * int_total    / total);
-  tty->print_cr("\t%9d (%4.1f%%) compiled",     comp_total,   100.0 * comp_total   / total);
-  tty->print_cr("\t%9d (100%%)  total",         total);
-  tty->print_cr("\t%9d (%4.1f%%) synchronized", synch_total,  100.0 * synch_total  / total);
-  tty->print_cr("\t%9d (%4.1f%%) final",        final_total,  100.0 * final_total  / total);
-  tty->print_cr("\t%9d (%4.1f%%) static",       static_total, 100.0 * static_total / total);
-  tty->print_cr("\t%9d (%4.1f%%) native",       nativ_total,  100.0 * nativ_total  / total);
-  tty->print_cr("\t%9d (%4.1f%%) accessor",     acces_total,  100.0 * acces_total  / total);
+  special_total = final_total + static_total +synch_total + native_total + access_total;
+  tty->print_cr("Invocations summary for %d methods:", collected_invoked_methods->length());
+  tty->print_cr("\t" UINT64_FORMAT_W(12) " (100%%)  total",           total);
+  tty->print_cr("\t" UINT64_FORMAT_W(12) " (%4.1f%%) |- interpreted", int_total,     100.0 * int_total    / total);
+  tty->print_cr("\t" UINT64_FORMAT_W(12) " (%4.1f%%) |- compiled",    comp_total,    100.0 * comp_total   / total);
+  tty->print_cr("\t" UINT64_FORMAT_W(12) " (%4.1f%%) |- special methods (interpreted and compiled)",
+                                                                         special_total, 100.0 * special_total/ total);
+  tty->print_cr("\t" UINT64_FORMAT_W(12) " (%4.1f%%)    |- synchronized",synch_total,   100.0 * synch_total  / total);
+  tty->print_cr("\t" UINT64_FORMAT_W(12) " (%4.1f%%)    |- final",       final_total,   100.0 * final_total  / total);
+  tty->print_cr("\t" UINT64_FORMAT_W(12) " (%4.1f%%)    |- static",      static_total,  100.0 * static_total / total);
+  tty->print_cr("\t" UINT64_FORMAT_W(12) " (%4.1f%%)    |- native",      native_total,  100.0 * native_total / total);
+  tty->print_cr("\t" UINT64_FORMAT_W(12) " (%4.1f%%)    |- accessor",    access_total,  100.0 * access_total / total);
   tty->cr();
   SharedRuntime::print_call_statistics(comp_total);
 }