Skip to content

Commit

Permalink
8317809: Insertion of free code blobs into code cache can be very slo…
Browse files Browse the repository at this point in the history
…w during class unloading

Reviewed-by: phh, adinn
Backport-of: 30817b742300f10f566e6aee3a8c1f8af4ab3083
  • Loading branch information
GoeLin committed May 3, 2024
1 parent a4d8d06 commit 3bb8eee
Show file tree
Hide file tree
Showing 28 changed files with 396 additions and 131 deletions.
2 changes: 1 addition & 1 deletion src/hotspot/share/classfile/classLoaderData.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -599,7 +599,7 @@ void ClassLoaderData::unload() {
free_deallocate_list_C_heap_structures();

// Clean up class dependencies and tell serviceability tools
// these classes are unloading. Must be called
// these classes are unloading. This must be called
// after erroneous classes are released.
classes_do(InstanceKlass::unload_class);

Expand Down
14 changes: 9 additions & 5 deletions src/hotspot/share/classfile/classLoaderData.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -186,12 +186,16 @@ class ClassLoaderData : public CHeapObj<mtClass> {
ClassLoaderData* next() const;
void unlink_next();

void set_unloading_next(ClassLoaderData* unloading_next);
ClassLoaderData* unloading_next() const;

ClassLoaderData(Handle h_class_loader, bool has_class_mirror_holder);

public:
~ClassLoaderData();

void set_unloading_next(ClassLoaderData* unloading_next);
ClassLoaderData* unloading_next() const;
void unload();

private:
// The CLD are not placed in the Heap, so the Card Table or
// the Mod Union Table can't be used to mark when CLD have modified oops.
// The CT and MUT bits saves this information for the whole class loader data.
Expand All @@ -203,11 +207,11 @@ class ClassLoaderData : public CHeapObj<mtClass> {
oop holder_no_keepalive() const;
oop holder() const;

void classes_do(void f(Klass* const));

private:
void unload();
bool keep_alive() const { return _keep_alive > 0; }

void classes_do(void f(Klass* const));
void loaded_classes_do(KlassClosure* klass_closure);
void classes_do(void f(InstanceKlass*));
void methods_do(void f(Method*));
Expand Down
26 changes: 7 additions & 19 deletions src/hotspot/share/classfile/classLoaderDataGraph.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
#include "classfile/moduleEntry.hpp"
#include "classfile/packageEntry.hpp"
#include "code/dependencyContext.hpp"
#include "gc/shared/classUnloadingContext.hpp"
#include "logging/log.hpp"
#include "logging/logStream.hpp"
#include "memory/allocation.inline.hpp"
Expand Down Expand Up @@ -124,7 +125,6 @@ void ClassLoaderDataGraph::walk_metadata_and_clean_metaspaces() {

// List head of all class loader data.
ClassLoaderData* volatile ClassLoaderDataGraph::_head = nullptr;
ClassLoaderData* ClassLoaderDataGraph::_unloading_head = nullptr;

bool ClassLoaderDataGraph::_should_clean_deallocate_lists = false;
bool ClassLoaderDataGraph::_safepoint_cleanup_needed = false;
Expand Down Expand Up @@ -342,11 +342,7 @@ void ClassLoaderDataGraph::loaded_classes_do(KlassClosure* klass_closure) {
}

void ClassLoaderDataGraph::classes_unloading_do(void f(Klass* const)) {
assert_locked_or_safepoint(ClassLoaderDataGraph_lock);
for (ClassLoaderData* cld = _unloading_head; cld != nullptr; cld = cld->unloading_next()) {
assert(cld->is_unloading(), "invariant");
cld->classes_do(f);
}
ClassUnloadingContext::context()->classes_unloading_do(f);
}

void ClassLoaderDataGraph::verify_dictionary() {
Expand Down Expand Up @@ -425,7 +421,8 @@ bool ClassLoaderDataGraph::do_unloading() {
} else {
// Found dead CLD.
loaders_removed++;
data->unload();

ClassUnloadingContext::context()->register_unloading_class_loader_data(data);

// Move dead CLD to unloading list.
if (prev != nullptr) {
Expand All @@ -435,8 +432,6 @@ bool ClassLoaderDataGraph::do_unloading() {
// The GC might be walking this concurrently
Atomic::store(&_head, data->next());
}
data->set_unloading_next(_unloading_head);
_unloading_head = data;
}
}

Expand Down Expand Up @@ -469,16 +464,9 @@ void ClassLoaderDataGraph::clean_module_and_package_info() {
}

void ClassLoaderDataGraph::purge(bool at_safepoint) {
ClassLoaderData* list = _unloading_head;
_unloading_head = nullptr;
ClassLoaderData* next = list;
bool classes_unloaded = false;
while (next != nullptr) {
ClassLoaderData* purge_me = next;
next = purge_me->unloading_next();
delete purge_me;
classes_unloaded = true;
}
ClassUnloadingContext::context()->purge_class_loader_data();

bool classes_unloaded = ClassUnloadingContext::context()->has_unloaded_classes();

Metaspace::purge(classes_unloaded);
if (classes_unloaded) {
Expand Down
2 changes: 0 additions & 2 deletions src/hotspot/share/classfile/classLoaderDataGraph.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,6 @@ class ClassLoaderDataGraph : public AllStatic {
private:
// All CLDs (except unlinked CLDs) can be reached by walking _head->_next->...
static ClassLoaderData* volatile _head;
// All unlinked CLDs
static ClassLoaderData* _unloading_head;

// Set if there's anything to purge in the deallocate lists or previous versions
// during a safepoint after class unloading in a full GC.
Expand Down
4 changes: 2 additions & 2 deletions src/hotspot/share/code/codeBlob.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ RuntimeBlob::RuntimeBlob(
void RuntimeBlob::free(RuntimeBlob* blob) {
assert(blob != nullptr, "caller must check for nullptr");
ThreadInVMfromUnknown __tiv; // get to VM state in case we block on CodeCache_lock
blob->flush();
blob->purge();
{
MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
CodeCache::free(blob);
Expand All @@ -173,7 +173,7 @@ void RuntimeBlob::free(RuntimeBlob* blob) {
MemoryService::track_code_cache_memory_usage();
}

void CodeBlob::flush() {
void CodeBlob::purge(bool free_code_cache_data) {
if (_oop_maps != nullptr) {
delete _oop_maps;
_oop_maps = nullptr;
Expand Down
2 changes: 1 addition & 1 deletion src/hotspot/share/code/codeBlob.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ class CodeBlob {
static unsigned int align_code_offset(int offset);

// Deletion
virtual void flush();
virtual void purge(bool free_code_cache_data = true);

// Typing
virtual bool is_buffer_blob() const { return false; }
Expand Down
36 changes: 4 additions & 32 deletions src/hotspot/share/code/codeCache.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
#include "compiler/compilerDefinitions.inline.hpp"
#include "compiler/oopMap.hpp"
#include "gc/shared/barrierSetNMethod.hpp"
#include "gc/shared/classUnloadingContext.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "jfr/jfrEvents.hpp"
#include "jvm_io.h"
Expand Down Expand Up @@ -606,7 +607,7 @@ void CodeCache::free(CodeBlob* cb) {

cb->~CodeBlob();
// Get heap for given CodeBlob and deallocate
get_code_heap(cb)->deallocate(cb);
heap->deallocate(cb);

assert(heap->blob_count() >= 0, "sanity check");
}
Expand Down Expand Up @@ -970,36 +971,8 @@ void CodeCache::purge_exception_caches() {
_exception_cache_purge_list = nullptr;
}

// Register an is_unloading nmethod to be flushed after unlinking
void CodeCache::register_unlinked(nmethod* nm) {
assert(nm->unlinked_next() == nullptr, "Only register for unloading once");
for (;;) {
// Only need acquire when reading the head, when the next
// pointer is walked, which it is not here.
nmethod* head = Atomic::load(&_unlinked_head);
nmethod* next = head != nullptr ? head : nm; // Self looped means end of list
nm->set_unlinked_next(next);
if (Atomic::cmpxchg(&_unlinked_head, head, nm) == head) {
break;
}
}
}

// Flush all the nmethods the GC unlinked
void CodeCache::flush_unlinked_nmethods() {
nmethod* nm = _unlinked_head;
_unlinked_head = nullptr;
size_t freed_memory = 0;
while (nm != nullptr) {
nmethod* next = nm->unlinked_next();
freed_memory += nm->total_size();
nm->flush();
if (next == nm) {
// Self looped means end of list
break;
}
nm = next;
}
// Restart compiler if possible and required..
void CodeCache::maybe_restart_compiler(size_t freed_memory) {

// Try to start the compiler again if we freed any memory
if (!CompileBroker::should_compile_new_jobs() && freed_memory != 0) {
Expand All @@ -1013,7 +986,6 @@ void CodeCache::flush_unlinked_nmethods() {
}

uint8_t CodeCache::_unloading_cycle = 1;
nmethod* volatile CodeCache::_unlinked_head = nullptr;

void CodeCache::increment_unloading_cycle() {
// 2-bit value (see IsUnloadingState in nmethod.cpp for details)
Expand Down
4 changes: 1 addition & 3 deletions src/hotspot/share/code/codeCache.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,6 @@ class CodeCache : AllStatic {
static TruncatedSeq _unloading_gc_intervals;
static TruncatedSeq _unloading_allocation_rates;
static volatile bool _unloading_threshold_gc_requested;
static nmethod* volatile _unlinked_head;

static ExceptionCache* volatile _exception_cache_purge_list;

Expand Down Expand Up @@ -213,8 +212,7 @@ class CodeCache : AllStatic {
// nmethod::is_cold.
static void arm_all_nmethods();

static void flush_unlinked_nmethods();
static void register_unlinked(nmethod* nm);
static void maybe_restart_compiler(size_t freed_memory);
static void do_unloading(bool unloading_occurred);
static uint8_t unloading_cycle() { return _unloading_cycle; }

Expand Down
2 changes: 1 addition & 1 deletion src/hotspot/share/code/compiledMethod.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ class CompiledMethod : public CodeBlob {

void* _gc_data;

virtual void flush() = 0;
virtual void purge(bool free_code_cache_data = true) = 0;

private:
DeoptimizationStatus deoptimization_status() const {
Expand Down
17 changes: 10 additions & 7 deletions src/hotspot/share/code/nmethod.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
#include "compiler/oopMap.inline.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetNMethod.hpp"
#include "gc/shared/classUnloadingContext.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "interpreter/bytecode.hpp"
#include "jvm.h"
Expand Down Expand Up @@ -639,7 +640,7 @@ nmethod::nmethod(
ByteSize basic_lock_sp_offset,
OopMapSet* oop_maps )
: CompiledMethod(method, "native nmethod", type, nmethod_size, sizeof(nmethod), code_buffer, offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false, true),
_unlinked_next(nullptr),
_is_unlinked(false),
_native_receiver_sp_offset(basic_lock_owner_sp_offset),
_native_basic_lock_sp_offset(basic_lock_sp_offset),
_is_unloading_state(0)
Expand Down Expand Up @@ -783,7 +784,7 @@ nmethod::nmethod(
#endif
)
: CompiledMethod(method, "nmethod", type, nmethod_size, sizeof(nmethod), code_buffer, offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false, true),
_unlinked_next(nullptr),
_is_unlinked(false),
_native_receiver_sp_offset(in_ByteSize(-1)),
_native_basic_lock_sp_offset(in_ByteSize(-1)),
_is_unloading_state(0)
Expand Down Expand Up @@ -1406,7 +1407,7 @@ bool nmethod::make_not_entrant() {

// For concurrent GCs, there must be a handshake between unlink and flush
void nmethod::unlink() {
if (_unlinked_next != nullptr) {
if (_is_unlinked) {
// Already unlinked. It can be invoked twice because concurrent code cache
// unloading might need to restart when inline cache cleaning fails due to
// running out of ICStubs, which can only be refilled at safepoints
Expand Down Expand Up @@ -1440,10 +1441,10 @@ void nmethod::unlink() {
// Register for flushing when it is safe. For concurrent class unloading,
// that would be after the unloading handshake, and for STW class unloading
// that would be when getting back to the VM thread.
CodeCache::register_unlinked(this);
ClassUnloadingContext::context()->register_unlinked_nmethod(this);
}

void nmethod::flush() {
void nmethod::purge(bool free_code_cache_data) {
MutexLocker ml(CodeCache_lock, Mutex::_no_safepoint_check_flag);

// completely deallocate this method
Expand All @@ -1466,8 +1467,10 @@ void nmethod::flush() {
Universe::heap()->unregister_nmethod(this);
CodeCache::unregister_old_nmethod(this);

CodeBlob::flush();
CodeCache::free(this);
CodeBlob::purge();
if (free_code_cache_data) {
CodeCache::free(this);
}
}

oop nmethod::oop_at(int index) const {
Expand Down
8 changes: 4 additions & 4 deletions src/hotspot/share/code/nmethod.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ class nmethod : public CompiledMethod {
address _verified_entry_point; // entry point without class check
address _osr_entry_point; // entry point for on stack replacement

nmethod* _unlinked_next;
bool _is_unlinked;

// Shared fields for all nmethod's
int _entry_bci; // != InvocationEntryBci if this nmethod is an on-stack replacement method
Expand Down Expand Up @@ -441,8 +441,8 @@ class nmethod : public CompiledMethod {
virtual bool is_unloading();
virtual void do_unloading(bool unloading_occurred);

nmethod* unlinked_next() const { return _unlinked_next; }
void set_unlinked_next(nmethod* next) { _unlinked_next = next; }
bool is_unlinked() const { return _is_unlinked; }
void set_is_unlinked() { assert(!_is_unlinked, "already unlinked"); _is_unlinked = true; }

#if INCLUDE_RTM_OPT
// rtm state accessing and manipulating
Expand Down Expand Up @@ -522,7 +522,7 @@ class nmethod : public CompiledMethod {
void unlink();

// Deallocate this nmethod - called by the GC
void flush();
void purge(bool free_code_cache_data = true);

// See comment at definition of _last_seen_on_stack
void mark_as_maybe_on_stack();
Expand Down
2 changes: 1 addition & 1 deletion src/hotspot/share/compiler/compileBroker.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1775,7 +1775,7 @@ bool CompileBroker::init_compiler_runtime() {
void CompileBroker::free_buffer_blob_if_allocated(CompilerThread* thread) {
BufferBlob* blob = thread->get_buffer_blob();
if (blob != nullptr) {
blob->flush();
blob->purge();
MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
CodeCache::free(blob);
}
Expand Down
34 changes: 29 additions & 5 deletions src/hotspot/share/gc/g1/g1CollectedHeap.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
#include "precompiled.hpp"
#include "classfile/classLoaderDataGraph.hpp"
#include "classfile/metadataOnStackMark.hpp"
#include "classfile/stringTable.hpp"
#include "classfile/systemDictionary.hpp"
#include "code/codeCache.hpp"
#include "code/icBuffer.hpp"
#include "compiler/oopMap.hpp"
Expand Down Expand Up @@ -74,6 +74,7 @@
#include "gc/g1/heapRegion.inline.hpp"
#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "gc/g1/heapRegionSet.inline.hpp"
#include "gc/shared/classUnloadingContext.hpp"
#include "gc/shared/concurrentGCBreakpoints.hpp"
#include "gc/shared/gcBehaviours.hpp"
#include "gc/shared/gcHeapSummary.hpp"
Expand Down Expand Up @@ -855,10 +856,6 @@ void G1CollectedHeap::verify_before_full_collection() {
}

void G1CollectedHeap::prepare_for_mutator_after_full_collection() {
// Delete metaspaces for unloaded class loaders and clean up loader_data graph
ClassLoaderDataGraph::purge(/*at_safepoint*/true);
DEBUG_ONLY(MetaspaceUtils::verify();)

// Prepare heap for normal collections.
assert(num_free_regions() == 0, "we should not have added any free regions");
rebuild_region_sets(false /* free_list_only */);
Expand Down Expand Up @@ -2596,6 +2593,33 @@ void G1CollectedHeap::complete_cleaning(bool class_unloading_occurred) {
workers()->run_task(&unlink_task);
}

void G1CollectedHeap::unload_classes_and_code(const char* description, BoolObjectClosure* is_alive, GCTimer* timer) {
GCTraceTime(Debug, gc, phases) debug(description, timer);

ClassUnloadingContext ctx(workers()->active_workers(),
false /* lock_codeblob_free_separately */);
{
CodeCache::UnlinkingScope scope(is_alive);
bool unloading_occurred = SystemDictionary::do_unloading(timer);
GCTraceTime(Debug, gc, phases) t("G1 Complete Cleaning", timer);
complete_cleaning(unloading_occurred);
}
{
GCTraceTime(Debug, gc, phases) t("Purge Unlinked NMethods", timer);
ctx.purge_nmethods();
}
{
GCTraceTime(Debug, gc, phases) t("Free Code Blobs", timer);
ctx.free_code_blobs();
}
{
GCTraceTime(Debug, gc, phases) t("Purge Class Loader Data", timer);
ClassLoaderDataGraph::purge(true /* at_safepoint */);
DEBUG_ONLY(MetaspaceUtils::verify();)
}
}


bool G1STWSubjectToDiscoveryClosure::do_object_b(oop obj) {
assert(obj != nullptr, "must not be null");
assert(_g1h->is_in_reserved(obj), "Trying to discover obj " PTR_FORMAT " not in heap", p2i(obj));
Expand Down

1 comment on commit 3bb8eee

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.