8243200: Shenandoah: Allow concurrent nmethod iteration

Reviewed-by: rkennke, shade

src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.hpp

@@ -28,14 +28,16 @@
 #include "code/codeCache.hpp"
 #include "gc/shenandoah/shenandoahSharedVariables.hpp"
 #include "gc/shenandoah/shenandoahLock.hpp"
-#include "gc/shenandoah/shenandoahNMethod.hpp"
 #include "gc/shenandoah/shenandoahPadding.hpp"
 #include "memory/allocation.hpp"
 #include "memory/iterator.hpp"
 #include "utilities/globalDefinitions.hpp"
 
 class ShenandoahHeap;
 class ShenandoahHeapRegion;
+class ShenandoahNMethodTable;
+class ShenandoahNMethodTableSnapshot;
+class WorkGang;
 
 class ShenandoahParallelCodeHeapIterator {
   friend class CodeCache;

src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp

@@ -318,20 +318,19 @@ void ShenandoahNMethod::assert_no_oops(nmethod* nm, bool allow_dead) {
 
 ShenandoahNMethodTable::ShenandoahNMethodTable() :
   _heap(ShenandoahHeap::heap()),
-  _size(minSize),
   _index(0),
-  _iteration_in_progress(false) {
-  _array = NEW_C_HEAP_ARRAY(ShenandoahNMethod*, _size, mtGC);
+  _itr_cnt(0) {
+  _list = new ShenandoahNMethodList(minSize);
 }
 
 ShenandoahNMethodTable::~ShenandoahNMethodTable() {
-  assert(_array != NULL, "Sanity");
-  FREE_C_HEAP_ARRAY(ShenandoahNMethod*, _array);
+  assert(_list != NULL, "Sanity");
+  _list->release();
 }
 
 void ShenandoahNMethodTable::register_nmethod(nmethod* nm) {
   assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held");
-  assert(_index >= 0 && _index <= _size, "Sanity");
+  assert(_index >= 0 && _index <= _list->size(), "Sanity");
 
   ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm);
   ShenandoahReentrantLocker data_locker(data != NULL ? data->lock() : NULL);
@@ -406,12 +405,12 @@ bool ShenandoahNMethodTable::contain(nmethod* nm) const {
 
 ShenandoahNMethod* ShenandoahNMethodTable::at(int index) const {
   assert(index >= 0 && index < _index, "Out of bound");
-  return _array[index];
+  return _list->at(index);
 }
 
 int ShenandoahNMethodTable::index_of(nmethod* nm) const {
   for (int index = 0; index < length(); index ++) {
-    if (_array[index]->nm() == nm) {
+    if (at(index)->nm() == nm) {
       return index;
     }
   }
@@ -420,14 +419,14 @@ int ShenandoahNMethodTable::index_of(nmethod* nm) const {
 
 void ShenandoahNMethodTable::remove(int idx) {
   shenandoah_assert_locked_or_safepoint(CodeCache_lock);
-  assert(!_iteration_in_progress, "Can not happen");
-  assert(_index >= 0 && _index <= _size, "Sanity");
+  assert(!iteration_in_progress(), "Can not happen");
+  assert(_index >= 0 && _index <= _list->size(), "Sanity");
 
   assert(idx >= 0 && idx < _index, "Out of bound");
-  ShenandoahNMethod* snm = _array[idx];
-
+  ShenandoahNMethod* snm = _list->at(idx);
+  ShenandoahNMethod* tmp = _list->at(_index - 1);
+  _list->set(idx, tmp);
   _index --;
-  _array[idx] = _array[_index];
 
   delete snm;
 }
@@ -441,48 +440,34 @@ void ShenandoahNMethodTable::wait_until_concurrent_iteration_done() {
 
 void ShenandoahNMethodTable::append(ShenandoahNMethod* snm) {
   if (is_full()) {
-    int new_size = 2 * _size;
-    ShenandoahNMethod** old_table = _array;
-
+    int new_size = 2 * _list->size();
     // Rebuild table and replace current one
     rebuild(new_size);
-
-    // An iteration is in progress over early snapshot,
-    // can not release the array until iteration is completed
-    if (!iteration_in_progress()) {
-      FREE_C_HEAP_ARRAY(ShenandoahNMethod*, old_table);
-    }
   }
 
-  _array[_index ++] = snm;
-  assert(_index >= 0 && _index <= _size, "Sanity");
+  _list->set(_index++,  snm);
+  assert(_index >= 0 && _index <= _list->size(), "Sanity");
 }
 
 void ShenandoahNMethodTable::rebuild(int size) {
-  ShenandoahNMethod** arr = NEW_C_HEAP_ARRAY(ShenandoahNMethod*, size, mtGC);
-  for (int index = 0; index < _index; index ++) {
-      arr[index] = _array[index];
-  }
-  _array = arr;
-  _size = size;
+  ShenandoahNMethodList* new_list = new ShenandoahNMethodList(size);
+  new_list->transfer(_list, _index);
+
+  // Release old list
+  _list->release();
+  _list = new_list;
 }
 
 ShenandoahNMethodTableSnapshot* ShenandoahNMethodTable::snapshot_for_iteration() {
-  assert(!iteration_in_progress(), "Already in progress");
-  _iteration_in_progress = true;
-
+  _itr_cnt++;
   return new ShenandoahNMethodTableSnapshot(this);
 }
 
 void ShenandoahNMethodTable::finish_iteration(ShenandoahNMethodTableSnapshot* snapshot) {
   assert(iteration_in_progress(), "Why we here?");
   assert(snapshot != NULL, "No snapshot");
-  _iteration_in_progress = false;
+  _itr_cnt--;
 
-  // Table has been rebuilt during iteration, free old table
-  if (snapshot->_array != _array) {
-    FREE_C_HEAP_ARRAY(ShenandoahNMethod*, snapshot->_array);
-  }
   delete snapshot;
 }
 
@@ -528,23 +513,61 @@ void ShenandoahNMethodTable::assert_nmethods_alive_and_correct() {
   assert_locked_or_safepoint(CodeCache_lock);
 
   for (int index = 0; index < length(); index ++) {
-    ShenandoahNMethod* m = _array[index];
+    ShenandoahNMethod* m = _list->at(index);
     // Concurrent unloading may have dead nmethods to be cleaned by sweeper
     if (m->is_unregistered()) continue;
     m->assert_alive_and_correct();
   }
 }
 #endif
 
+
+ShenandoahNMethodList::ShenandoahNMethodList(int size) :
+  _size(size), _ref_count(1) {
+  _list = NEW_C_HEAP_ARRAY(ShenandoahNMethod*, size, mtGC);
+}
+
+ShenandoahNMethodList::~ShenandoahNMethodList() {
+  assert(_list != NULL, "Sanity");
+  assert(_ref_count == 0, "Must be");
+  FREE_C_HEAP_ARRAY(ShenandoahNMethod*, _list);
+}
+
+void ShenandoahNMethodList::transfer(ShenandoahNMethodList* const list, int limit) {
+  assert(limit <= size(), "Sanity");
+  ShenandoahNMethod** old_list = list->list();
+  for (int index = 0; index < limit; index++) {
+    _list[index] = old_list[index];
+  }
+}
+
+ShenandoahNMethodList* ShenandoahNMethodList::acquire() {
+  assert(CodeCache_lock->owned_by_self(), "Lock must be held");
+  _ref_count++;
+  return this;
+}
+
+void ShenandoahNMethodList::release() {
+  assert(CodeCache_lock->owned_by_self(), "Lock must be held");
+  _ref_count--;
+  if (_ref_count == 0) {
+    delete this;
+  }
+}
+
 ShenandoahNMethodTableSnapshot::ShenandoahNMethodTableSnapshot(ShenandoahNMethodTable* table) :
-  _heap(ShenandoahHeap::heap()), _table(table), _array(table->_array), _length(table->_index), _claimed(0) {
+  _heap(ShenandoahHeap::heap()), _list(table->_list->acquire()), _limit(table->_index), _claimed(0) {
+}
+
+ShenandoahNMethodTableSnapshot::~ShenandoahNMethodTableSnapshot() {
+  _list->release();
 }
 
 void ShenandoahNMethodTableSnapshot::concurrent_nmethods_do(NMethodClosure* cl) {
   size_t stride = 256; // educated guess
 
-  ShenandoahNMethod** list = _array;
-  size_t max = (size_t)_length;
+  ShenandoahNMethod** list = _list->list();
+  size_t max = (size_t)_limit;
   while (_claimed < max) {
     size_t cur = Atomic::fetch_and_add(&_claimed, stride);
     size_t start = cur;

src/hotspot/share/gc/shenandoah/shenandoahNMethod.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -83,21 +83,48 @@ class ShenandoahNMethod : public CHeapObj<mtGC> {
 
 class ShenandoahNMethodTable;
 
+// ShenandoahNMethodList holds registered nmethod data. The list is reference counted.
+class ShenandoahNMethodList : public CHeapObj<mtGC> {
+private:
+  ShenandoahNMethod** _list;
+  const int           _size;
+  uint                _ref_count;
+
+private:
+  ~ShenandoahNMethodList();
+
+public:
+  ShenandoahNMethodList(int size);
+
+  // Reference counting with CoceCache_lock held
+  ShenandoahNMethodList* acquire();
+  void release();
+
+  // Transfer content from other list to 'this' list, up to the limit
+  void transfer(ShenandoahNMethodList* const other, int limit);
+
+  inline int size() const;
+  inline ShenandoahNMethod** list() const;
+  inline ShenandoahNMethod* at(int index) const;
+  inline void set(int index, ShenandoahNMethod* snm);
+};
+
 // An opaque snapshot of current nmethod table for iteration
 class ShenandoahNMethodTableSnapshot : public CHeapObj<mtGC> {
   friend class ShenandoahNMethodTable;
 private:
   ShenandoahHeap* const       _heap;
-  ShenandoahNMethodTable*     _table;
-  ShenandoahNMethod** const   _array;
-  const int                   _length;
+  ShenandoahNMethodList*      _list;
+  /* snapshot iteration limit */
+  int                         _limit;
 
   shenandoah_padding(0);
   volatile size_t       _claimed;
   shenandoah_padding(1);
 
 public:
   ShenandoahNMethodTableSnapshot(ShenandoahNMethodTable* table);
+  ~ShenandoahNMethodTableSnapshot();
 
   template<bool CSET_FILTER>
   void parallel_blobs_do(CodeBlobClosure *f);
@@ -112,12 +139,12 @@ class ShenandoahNMethodTable : public CHeapObj<mtGC> {
     minSize = 1024
   };
 
-  ShenandoahHeap* const _heap;
-  ShenandoahNMethod**   _array;
-  int                   _size;
-  int                   _index;
-  ShenandoahLock        _lock;
-  bool                  _iteration_in_progress;
+  ShenandoahHeap* const  _heap;
+  ShenandoahNMethodList* _list;
+
+  int                    _index;
+  ShenandoahLock         _lock;
+  int                    _itr_cnt;
 
 public:
   ShenandoahNMethodTable();
@@ -140,8 +167,8 @@ class ShenandoahNMethodTable : public CHeapObj<mtGC> {
   void rebuild(int size);
 
   bool is_full() const {
-    assert(_index <= _size, "Sanity");
-    return _index == _size;
+    assert(_index <= _list->size(), "Sanity");
+    return _index == _list->size();
   }
 
   ShenandoahNMethod* at(int index) const;

src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp

@@ -79,16 +79,35 @@ ShenandoahReentrantLock* ShenandoahNMethod::lock_for_nmethod(nmethod* nm) {
 }
 
 bool ShenandoahNMethodTable::iteration_in_progress() const {
-  return _iteration_in_progress;
+  shenandoah_assert_locked_or_safepoint(CodeCache_lock);
+  return _itr_cnt > 0;
+}
+
+int ShenandoahNMethodList::size() const {
+  return _size;
+}
+
+ShenandoahNMethod* ShenandoahNMethodList::at(int index) const {
+  assert(index < size(), "Index out of bound");
+  return _list[index];
+}
+
+void ShenandoahNMethodList::set(int index, ShenandoahNMethod* snm) {
+  assert(index < size(), "Index out of bound");
+  _list[index] = snm;
+}
+
+ShenandoahNMethod** ShenandoahNMethodList::list() const {
+  return _list;
 }
 
 template<bool CSET_FILTER>
 void ShenandoahNMethodTableSnapshot::parallel_blobs_do(CodeBlobClosure *f) {
   size_t stride = 256; // educated guess
 
-  ShenandoahNMethod** const list = _array;
+  ShenandoahNMethod** const list = _list->list();
 
-  size_t max = (size_t)_length;
+  size_t max = (size_t)_limit;
   while (_claimed < max) {
     size_t cur = Atomic::fetch_and_add(&_claimed, stride);
     size_t start = cur;