Skip to content
Permalink
Browse files
8255234: ZGC: Bulk allocate forwarding data structures
Co-authored-by: Albert Mingkun Yang <ayang@openjdk.org>
Co-authored-by: Per Liden <pliden@openjdk.org>
Reviewed-by: ayang, stefank
  • Loading branch information
pliden and albertnetymk committed Oct 28, 2020
1 parent b7d483c commit 2c9dfc73f92bc8d1c38b4f2f595194fcc623aa11
Showing 16 changed files with 283 additions and 68 deletions.
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,8 +32,12 @@ class ZAttachedArray {
const size_t _length;

static size_t object_size();
static size_t array_size(size_t length);

public:
template <typename Allocator>
static void* alloc(Allocator* allocator, size_t length);

static void* alloc(size_t length);
static void free(ObjectT* obj);

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -34,13 +34,35 @@ inline size_t ZAttachedArray<ObjectT, ArrayT>::object_size() {
}

template <typename ObjectT, typename ArrayT>
inline void* ZAttachedArray<ObjectT, ArrayT>::alloc(size_t length) {
const size_t array_size = sizeof(ArrayT) * length;
char* const addr = AllocateHeap(object_size() + array_size, mtGC);
::new (addr + object_size()) ArrayT[length];
inline size_t ZAttachedArray<ObjectT, ArrayT>::array_size(size_t length) {
return sizeof(ArrayT) * length;
}

template <typename ObjectT, typename ArrayT>
template <typename Allocator>
inline void* ZAttachedArray<ObjectT, ArrayT>::alloc(Allocator* allocator, size_t length) {
// Allocate memory for object and array
const size_t size = object_size() + array_size(length);
void* const addr = allocator->alloc(size);

// Placement new array
void* const array_addr = reinterpret_cast<char*>(addr) + object_size();
::new (array_addr) ArrayT[length];

// Return pointer to object
return addr;
}

template <typename ObjectT, typename ArrayT>
inline void* ZAttachedArray<ObjectT, ArrayT>::alloc(size_t length) {
struct Allocator {
void* alloc(size_t size) const {
return AllocateHeap(size, mtGC);
}
} allocator;
return alloc(&allocator, length);
}

template <typename ObjectT, typename ArrayT>
inline void ZAttachedArray<ObjectT, ArrayT>::free(ObjectT* obj) {
FreeHeap(obj);
@@ -23,31 +23,7 @@

#include "precompiled.hpp"
#include "gc/z/zForwarding.inline.hpp"
#include "gc/z/zPage.inline.hpp"
#include "utilities/debug.hpp"
#include "utilities/powerOfTwo.hpp"

ZForwarding* ZForwarding::create(ZPage* page) {
// Allocate table for linear probing. The size of the table must be
// a power of two to allow for quick and inexpensive indexing/masking.
// The table is sized to have a load factor of 50%, i.e. sized to have
// double the number of entries actually inserted.
assert(page->live_objects() > 0, "Invalid value");
const size_t nentries = round_up_power_of_2(page->live_objects() * 2);
return ::new (AttachedArray::alloc(nentries)) ZForwarding(page, nentries);
}

void ZForwarding::destroy(ZForwarding* forwarding) {
AttachedArray::free(forwarding);
}

ZForwarding::ZForwarding(ZPage* page, size_t nentries) :
_virtual(page->virtual_memory()),
_object_alignment_shift(page->object_alignment_shift()),
_entries(nentries),
_page(page),
_refcount(1),
_pinned(false) {}

void ZForwarding::verify() const {
guarantee(_refcount > 0, "Invalid refcount");
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,7 @@
#include "gc/z/zForwardingEntry.hpp"
#include "gc/z/zVirtualMemory.hpp"

class ZForwardingAllocator;
class ZPage;

typedef size_t ZForwardingCursor;
@@ -57,8 +58,8 @@ class ZForwarding {
ZForwarding(ZPage* page, size_t nentries);

public:
static ZForwarding* create(ZPage* page);
static void destroy(ZForwarding* forwarding);
static uint32_t nentries(const ZPage* page);
static ZForwarding* alloc(ZForwardingAllocator* allocator, ZPage* page);

uintptr_t start() const;
size_t size() const;
@@ -26,11 +26,38 @@

#include "gc/z/zAttachedArray.inline.hpp"
#include "gc/z/zForwarding.hpp"
#include "gc/z/zForwardingAllocator.inline.hpp"
#include "gc/z/zHash.inline.hpp"
#include "gc/z/zHeap.hpp"
#include "gc/z/zPage.inline.hpp"
#include "gc/z/zVirtualMemory.inline.hpp"
#include "runtime/atomic.hpp"
#include "utilities/debug.hpp"
#include "utilities/powerOfTwo.hpp"

inline uint32_t ZForwarding::nentries(const ZPage* page) {
// The number returned by the function is used to size the hash table of
// forwarding entries for this page. This hash table uses linear probing.
// The size of the table must be a power of two to allow for quick and
// inexpensive indexing/masking. The table is also sized to have a load
// factor of 50%, i.e. sized to have double the number of entries actually
// inserted, to allow for good lookup/insert performance.
return round_up_power_of_2(page->live_objects() * 2);
}

inline ZForwarding* ZForwarding::alloc(ZForwardingAllocator* allocator, ZPage* page) {
const size_t nentries = ZForwarding::nentries(page);
void* const addr = AttachedArray::alloc(allocator, nentries);
return ::new (addr) ZForwarding(page, nentries);
}

inline ZForwarding::ZForwarding(ZPage* page, size_t nentries) :
_virtual(page->virtual_memory()),
_object_alignment_shift(page->object_alignment_shift()),
_entries(nentries),
_page(page),
_refcount(1),
_pinned(false) {}

inline uintptr_t ZForwarding::start() const {
return _virtual.start();
@@ -0,0 +1,40 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/

#include "precompiled.hpp"
#include "gc/z/zForwardingAllocator.hpp"
#include "memory/allocation.inline.hpp"

ZForwardingAllocator::ZForwardingAllocator() :
_start(NULL),
_end(NULL),
_top(NULL) {}

ZForwardingAllocator::~ZForwardingAllocator() {
FREE_C_HEAP_ARRAY(char, _start);
}

void ZForwardingAllocator::reset(size_t size) {
_start = _top = REALLOC_C_HEAP_ARRAY(char, _start, size, mtGC);
_end = _start + size;
}
@@ -0,0 +1,46 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/

#ifndef SHARE_GC_Z_ZFORWARDINGALLOCATOR_HPP
#define SHARE_GC_Z_ZFORWARDINGALLOCATOR_HPP

#include "utilities/globalDefinitions.hpp"

class ZForwardingAllocator {
private:
char* _start;
char* _end;
char* _top;

public:
ZForwardingAllocator();
~ZForwardingAllocator();

void reset(size_t size);
size_t size() const;
bool is_full() const;

void* alloc(size_t size);
};

#endif // SHARE_GC_Z_ZFORWARDINGALLOCATOR_HPP
@@ -0,0 +1,45 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/

#ifndef SHARE_GC_Z_ZFORWARDINGALLOCATOR_INLINE_HPP
#define SHARE_GC_Z_ZFORWARDINGALLOCATOR_INLINE_HPP

#include "gc/z/zForwardingAllocator.hpp"
#include "utilities/debug.hpp"

inline size_t ZForwardingAllocator::size() const {
return _end - _start;
}

inline bool ZForwardingAllocator::is_full() const {
return _top == _end;
}

inline void* ZForwardingAllocator::alloc(size_t size) {
char* const addr = _top;
_top += size;
assert(_top <= _end, "Allocation should never fail");
return addr;
}

#endif // SHARE_GC_Z_ZFORWARDINGALLOCATOR_INLINE_HPP
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -22,35 +22,53 @@
*/

#include "precompiled.hpp"
#include "gc/z/zForwarding.hpp"
#include "gc/z/zForwarding.inline.hpp"
#include "gc/z/zForwardingAllocator.inline.hpp"
#include "gc/z/zRelocationSet.hpp"
#include "gc/z/zStat.hpp"
#include "memory/allocation.hpp"
#include "utilities/debug.hpp"

ZRelocationSet::ZRelocationSet() :
_allocator(),
_forwardings(NULL),
_nforwardings(0) {}

void ZRelocationSet::populate(ZPage* const* group0, size_t ngroup0,
ZPage* const* group1, size_t ngroup1) {
_nforwardings = ngroup0 + ngroup1;
_forwardings = REALLOC_C_HEAP_ARRAY(ZForwarding*, _forwardings, _nforwardings, mtGC);
void ZRelocationSet::populate(ZPage* const* small, size_t nsmall,
ZPage* const* medium, size_t nmedium,
size_t forwarding_entries) {
// Set relocation set length
_nforwardings = nsmall + nmedium;

// Initialize forwarding allocator to have room for the
// relocation set, all forwardings, and all forwarding entries.
const size_t relocation_set_size = _nforwardings * sizeof(ZForwarding*);
const size_t forwardings_size = _nforwardings * sizeof(ZForwarding);
const size_t forwarding_entries_size = forwarding_entries * sizeof(ZForwardingEntry);
_allocator.reset(relocation_set_size + forwardings_size + forwarding_entries_size);

// Allocate relocation set
_forwardings = new (_allocator.alloc(relocation_set_size)) ZForwarding*[_nforwardings];

// Populate relocation set array
size_t j = 0;

// Populate group 0
for (size_t i = 0; i < ngroup0; i++) {
_forwardings[j++] = ZForwarding::create(group0[i]);
// Populate medium pages
for (size_t i = 0; i < nmedium; i++) {
_forwardings[j++] = ZForwarding::alloc(&_allocator, medium[i]);
}

// Populate group 1
for (size_t i = 0; i < ngroup1; i++) {
_forwardings[j++] = ZForwarding::create(group1[i]);
// Populate small pages
for (size_t i = 0; i < nsmall; i++) {
_forwardings[j++] = ZForwarding::alloc(&_allocator, small[i]);
}

assert(_allocator.is_full(), "Should be full");

// Update statistics
ZStatRelocation::set_at_populate_relocation_set(_allocator.size());
}

void ZRelocationSet::reset() {
for (size_t i = 0; i < _nforwardings; i++) {
ZForwarding::destroy(_forwardings[i]);
_forwardings[i] = NULL;
}
_nforwardings = 0;
}

1 comment on commit 2c9dfc7

@bridgekeeper
Copy link

@bridgekeeper bridgekeeper bot commented on 2c9dfc7 Oct 28, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.