8275415: Prepare Leak Profiler for Lilliput

Reviewed-by: rkennke

src/hotspot/share/jfr/leakprofiler/chains/edgeStore.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,8 +25,10 @@
 #include "precompiled.hpp"
 #include "jfr/leakprofiler/chains/edgeStore.hpp"
 #include "jfr/leakprofiler/chains/edgeUtils.hpp"
+#include "jfr/leakprofiler/sampling/objectSample.hpp"
 #include "jfr/leakprofiler/utilities/unifiedOopRef.inline.hpp"
 #include "oops/oop.inline.hpp"
+#include "runtime/safepoint.hpp"
 
 StoredEdge::StoredEdge(const Edge* parent, UnifiedOopRef reference) : Edge(parent, reference), _gc_root_id(0), _skip_length(0) {}
 
@@ -36,15 +38,6 @@ StoredEdge::StoredEdge(const StoredEdge& edge) : Edge(edge), _gc_root_id(edge._g
 
 traceid EdgeStore::_edge_id_counter = 0;
 
-EdgeStore::EdgeStore() : _edges(NULL) {
-  _edges = new EdgeHashTable(this);
-}
-
-EdgeStore::~EdgeStore() {
-  assert(_edges != NULL, "invariant");
-  delete _edges;
-}
-
 bool EdgeStore::is_empty() const {
   return !_edges->has_entries();
 }
@@ -224,15 +217,80 @@ bool EdgeStore::put_edges(StoredEdge** previous, const Edge** current, size_t li
   return NULL == *current;
 }
 
-// Install the immediate edge into the mark word of the leak candidate object
+static GrowableArray<const StoredEdge*>* _leak_context_edges = nullptr;
+
+EdgeStore::EdgeStore() : _edges(new EdgeHashTable(this)) {}
+
+EdgeStore::~EdgeStore() {
+  assert(_edges != NULL, "invariant");
+  delete _edges;
+  delete _leak_context_edges;
+  _leak_context_edges = nullptr;
+}
+
+static int leak_context_edge_idx(const ObjectSample* sample) {
+  assert(sample != nullptr, "invariant");
+  return static_cast<int>(sample->object()->mark().value()) >> markWord::lock_bits;
+}
+
+bool EdgeStore::has_leak_context(const ObjectSample* sample) const {
+  return leak_context_edge_idx(sample) != 0;
+}
+
+const StoredEdge* EdgeStore::get(const ObjectSample* sample) const {
+  assert(sample != nullptr, "invariant");
+  if (_leak_context_edges != nullptr) {
+    assert(SafepointSynchronize::is_at_safepoint(), "invariant");
+    const int idx = leak_context_edge_idx(sample);
+    if (idx > 0) {
+      return _leak_context_edges->at(idx);
+    }
+  }
+  return get(UnifiedOopRef::encode_in_native(sample->object_addr()));
+}
+
+#ifdef ASSERT
+// max_idx to ensure idx fit in lower 32-bits of markword together with lock bits.
+static constexpr const int max_idx =  right_n_bits(32 - markWord::lock_bits);
+
+static void store_idx_precondition(oop sample_object, int idx) {
+  assert(sample_object != NULL, "invariant");
+  assert(sample_object->mark().is_marked(), "invariant");
+  assert(idx > 0, "invariant");
+  assert(idx <= max_idx, "invariant");
+}
+#endif
+
+static void store_idx_in_markword(oop sample_object, int idx) {
+  DEBUG_ONLY(store_idx_precondition(sample_object, idx);)
+  const markWord idx_mark_word(sample_object->mark().value() | idx << markWord::lock_bits);
+  sample_object->set_mark(idx_mark_word);
+  assert(sample_object->mark().is_marked(), "must still be marked");
+}
+
+static const int initial_size = 64;
+
+static int save(const StoredEdge* edge) {
+  assert(edge != nullptr, "invariant");
+  if (_leak_context_edges == nullptr) {
+    _leak_context_edges = new (ResourceObj::C_HEAP, mtTracing)GrowableArray<const StoredEdge*>(initial_size, mtTracing);
+    _leak_context_edges->append(nullptr); // next idx now at 1, for disambiguation in markword.
+  }
+  return _leak_context_edges->append(edge);
+}
+
+// We associate the leak context edge with the leak candidate object by saving the
+// edge in an array and storing the array idx (shifted) into the markword of the candidate object.
+static void associate_with_candidate(const StoredEdge* leak_context_edge) {
+  assert(leak_context_edge != nullptr, "invariant");
+  store_idx_in_markword(leak_context_edge->pointee(), save(leak_context_edge));
+}
+
 StoredEdge* EdgeStore::associate_leak_context_with_candidate(const Edge* edge) {
   assert(edge != NULL, "invariant");
   assert(!contains(edge->reference()), "invariant");
   StoredEdge* const leak_context_edge = put(edge->reference());
-  oop sample_object = edge->pointee();
-  assert(sample_object != NULL, "invariant");
-  assert(sample_object->mark().is_marked(), "invariant");
-  sample_object->set_mark(markWord::from_pointer(leak_context_edge));
+  associate_with_candidate(leak_context_edge);
   return leak_context_edge;
 }
 

src/hotspot/share/jfr/leakprofiler/chains/edgeStore.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,6 +31,7 @@
 #include "memory/allocation.hpp"
 
 typedef u8 traceid;
+class ObjectSample;
 
 class StoredEdge : public Edge {
  private:
@@ -79,6 +80,7 @@ class EdgeStore : public CHeapObj<mtTracing> {
   void on_unlink(EdgeEntry* entry);
 
   StoredEdge* get(UnifiedOopRef reference) const;
+  const StoredEdge* get(const ObjectSample* sample) const;
   StoredEdge* put(UnifiedOopRef reference);
   traceid gc_root_id(const Edge* edge) const;
 
@@ -90,6 +92,7 @@ class EdgeStore : public CHeapObj<mtTracing> {
   void store_gc_root_id_in_leak_context_edge(StoredEdge* leak_context_edge, const Edge* root) const;
   StoredEdge* link_new_edge(StoredEdge** previous, const Edge** current);
   void link_with_existing_chain(const StoredEdge* current_stored, StoredEdge** previous, size_t previous_length);
+  bool has_leak_context(const ObjectSample* sample) const;
 
   template <typename T>
   void iterate(T& functor) const { _edges->iterate_value<T>(functor); }

src/hotspot/share/jfr/leakprofiler/checkpoint/eventEmitter.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2021, Datadog, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -121,7 +121,7 @@ void EventEmitter::link_sample_with_edge(const ObjectSample* sample, EdgeStore*
   assert(!sample->is_dead(), "invariant");
   assert(edge_store != NULL, "invariant");
   if (SafepointSynchronize::is_at_safepoint()) {
-    if (!sample->object()->mark().is_marked()) {
+    if (edge_store->has_leak_context(sample)) {
       // Associated with an edge (chain) already during heap traversal.
       return;
     }
@@ -138,21 +138,12 @@ void EventEmitter::write_event(const ObjectSample* sample, EdgeStore* edge_store
   assert(edge_store != NULL, "invariant");
   assert(_jfr_thread_local != NULL, "invariant");
 
-  traceid gc_root_id = 0;
-  const Edge* edge = NULL;
-  if (SafepointSynchronize::is_at_safepoint()) {
-    if (!sample->object()->mark().is_marked()) {
-      edge = (const Edge*)(sample->object())->mark().to_pointer();
-    }
-  }
-  if (edge == NULL) {
-    edge = edge_store->get(UnifiedOopRef::encode_in_native(sample->object_addr()));
-  } else {
-    gc_root_id = edge_store->gc_root_id(edge);
-  }
+  const StoredEdge* const edge = edge_store->get(sample);
   assert(edge != NULL, "invariant");
+  assert(edge->pointee() == sample->object(), "invariant");
   const traceid object_id = edge_store->get_id(edge);
   assert(object_id != 0, "invariant");
+  const traceid gc_root_id = edge->gc_root_id();
 
   Tickspan object_age = Ticks(_start_time.value()) - sample->allocation_time();