Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Read class from object header #12

Closed
wants to merge 55 commits into from
Closed
Changes from all commits
Commits
Show all changes
55 commits
Select commit Hold shift + click to select a range
07d96e7
Implement sliding forwarding scheme that preserves upper header bits
rkennke Jun 10, 2021
7b12373
32bit support
rkennke Jun 10, 2021
b754981
Build fix
rkennke Jun 11, 2021
f2c1883
Windows build fix
rkennke Jun 11, 2021
07fc51f
More build fixes and cleanups
rkennke Jun 14, 2021
f6b2426
Fix & vs == evaluation order
rkennke Jun 14, 2021
4d8cf44
Make number of region bits configurable via template
rkennke Jun 16, 2021
3e10d76
Some const-ness improvements
rkennke Jun 16, 2021
69a7d0e
Fix region_contains()
rkennke Jun 16, 2021
171a2fc
Remove include of precompiled.hpp
rkennke Jun 16, 2021
020b249
Use sliding forwarding in Serial GC
rkennke Jun 14, 2021
b65c222
Build fixes for templated SlidingForwarding
rkennke Jun 16, 2021
7c15762
Const-ness fixes for sliding forwarding in Serial GC
rkennke Jun 16, 2021
9348d59
SlidingForwarding support for G1GC
rkennke Jun 16, 2021
88d6cac
Fixes for G1
rkennke Jun 16, 2021
e4a27ba
Implement self-forwarding of objects that preserves header bits
rkennke Jun 28, 2021
92c2f1a
Merge branch 'master' into sliding-forwarding
rkennke Jun 29, 2021
fb91016
Merge branch 'self-forwarding' into sliding-and-self-forwarding
rkennke Jun 30, 2021
83a607f
Disable G1 serial full-GC, simplify sliding forwarding
rkennke Jul 5, 2021
a1cd1aa
Merge branch 'sliding-forwarding' into sliding-and-self-forwarding
rkennke Jul 5, 2021
04da65e
Keep self-forwarded header bit free for self-forwarding
rkennke Jul 5, 2021
de1d4e0
Merge branch 'master' into sliding-forwarding
rkennke Jul 5, 2021
b910229
Merge branch 'sliding-forwarding' into sliding-and-self-forwarding
rkennke Jul 5, 2021
7d41e93
Some cosmetic fixes
rkennke Jul 5, 2021
d23277f
Merge branch 'sliding-forwarding' into sliding-and-self-forwarding
rkennke Jul 5, 2021
49a613d
Use Klass* from header
rkennke Jul 7, 2021
294012a
Merge
rkennke Jul 9, 2021
de67373
Implement klass_or_null() variants
rkennke Jul 12, 2021
9a8d3b4
Several fixes
rkennke Jul 13, 2021
c578fc9
Avoid re-reading klass for is_phantom check
rkennke Jul 13, 2021
ad1d086
Revert unnecessary change
rkennke Jul 13, 2021
51058ef
Better klass preservation in self-forwarding
rkennke Jul 13, 2021
8457d97
Revert more unnecessary changes
rkennke Jul 13, 2021
50d2c36
Revert more unnecessary changes
rkennke Jul 13, 2021
287de4c
Merge branch 'master' into klass-from-header
rkennke Jul 16, 2021
bab711d
Always use acquire in read_stable_mark()
rkennke Jul 20, 2021
f5f641e
Added some debug output where ZGC fails
rkennke Jul 20, 2021
4bcfeb4
Some ZGC debugging
rkennke Jul 23, 2021
8568354
Merge branch 'master' into klass-from-header
rkennke Jul 26, 2021
b2d5676
Simplify OS::stable_mark()
rkennke Jul 27, 2021
0e444c4
Revert Shenandoah changes
rkennke Jul 27, 2021
e568ee0
Revert one more Shenandoah change
rkennke Jul 27, 2021
0af7fe2
Revert ZGC changes
rkennke Jul 27, 2021
f807683
Simplify oopDesc::klass() and related methods
rkennke Jul 27, 2021
11a54eb
Fix 32bit build
rkennke Jul 27, 2021
21a8cff
Simplify fetching klass from header at safepoint
rkennke Jul 27, 2021
8cbf853
Revert unnecessary change
rkennke Jul 27, 2021
a826d4b
Removed commented-out code
rkennke Jul 27, 2021
f98e354
Merge branch 'master' into klass-from-header
rkennke Jul 27, 2021
3203a50
Disable Shenandoah and ZGC in GA jobs
rkennke Jul 28, 2021
1f9e5ae
Add missing new markWord.inline.hpp
rkennke Jul 28, 2021
10aa5fc
Assert +UseCompressedClassPointers
rkennke Sep 16, 2021
f51ee5d
Re-shape ObjectSynchronizer::stable_mark() and add assert
rkennke Sep 16, 2021
be38608
Formatting changes
rkennke Sep 16, 2021
f9c5840
Rename oopDesc::narrow_klass() to narrow_klass_legacy()
rkennke Sep 16, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
@@ -180,6 +180,7 @@ jobs:
- name: Configure
run: >
bash configure
--with-jvm-features=-shenandoahgc,-zgc
--with-conf-name=linux-x64
${{ matrix.flags }}
--with-version-opt=${GITHUB_ACTOR}-${GITHUB_SHA}
@@ -545,6 +546,7 @@ jobs:
- name: Configure
run: >
bash configure
--with-jvm-features=-shenandoahgc,-zgc
--with-conf-name=linux-${{ matrix.gnu-arch }}-hotspot
${{ matrix.flags }}
${{ env.cross_flags }}
@@ -644,6 +646,7 @@ jobs:
- name: Configure
run: >
bash configure
--with-jvm-features=-shenandoahgc,-zgc
--with-conf-name=linux-x86
--with-target-bits=32
${{ matrix.flags }}
@@ -920,6 +923,7 @@ jobs:
$env:Path = $env:Path -split ";" -match "C:\\Windows|PowerShell|cygwin" -join ";" ;
$env:BOOT_JDK = cygpath "$HOME/bootjdk/$env:BOOT_JDK_VERSION" ;
& bash configure
--with-jvm-features=-shenandoahgc,-zgc
--with-conf-name=windows-aarch64
--with-msvc-toolset-version=14.29
--openjdk-target=aarch64-unknown-cygwin
@@ -1033,6 +1037,7 @@ jobs:
$env:JT_HOME = cygpath "$HOME/jtreg" ;
$env:GTEST = cygpath "$env:GITHUB_WORKSPACE/gtest" ;
& bash configure
--with-jvm-features=-shenandoahgc,-zgc
--with-conf-name=windows-x64
--with-msvc-toolset-version=14.28
${{ matrix.flags }}
@@ -1340,6 +1345,7 @@ jobs:
- name: Configure
run: >
bash configure
--with-jvm-features=-shenandoahgc,-zgc
--with-conf-name=macos-x64
${{ matrix.flags }}
--with-version-opt=${GITHUB_ACTOR}-${GITHUB_SHA}
@@ -1442,6 +1448,7 @@ jobs:
- name: Configure
run: >
bash configure
--with-jvm-features=-shenandoahgc,-zgc
--with-conf-name=macos-aarch64
--openjdk-target=aarch64-apple-darwin
${{ matrix.flags }}
@@ -116,6 +116,9 @@ class RemoveSelfForwardPtrObjClosure: public ObjectClosure {
// The object failed to move.

zap_dead_objects(_last_forwarded_object_end, obj_addr);

PreservedMarks::init_forwarded_mark(obj);

// We consider all objects that we find self-forwarded to be
// live. What we'll do is that we'll update the prev marking
// info so that they are all under PTAMS and explicitly marked.
@@ -138,7 +141,6 @@ class RemoveSelfForwardPtrObjClosure: public ObjectClosure {
size_t obj_size = obj->size();

_marked_bytes += (obj_size * HeapWordSize);
PreservedMarks::init_forwarded_mark(obj);

// While we were processing RSet buffers during the collection,
// we actually didn't scan any cards on the collection set,
@@ -158,6 +160,14 @@ class RemoveSelfForwardPtrObjClosure: public ObjectClosure {
_last_forwarded_object_end = obj_end;
_hr->cross_threshold(obj_addr, obj_end);
}
#ifdef _LP64
else if (obj->is_forwarded()) {
// Restore klass so that we can safely iterate.
// TODO: This could probably be built more efficiently into the iterator.
Klass* klass = obj->forwardee()->klass();
obj->set_mark(markWord::prototype().set_klass(klass));
}
#endif
}

// Fill the memory area from start to end with filler objects, and update the BOT
@@ -218,7 +218,6 @@ void G1ParScanThreadState::do_partial_array(PartialArrayScanTask task) {
oop from_obj = task.to_source_array();

assert(_g1h->is_in_reserved(from_obj), "must be in heap.");
assert(from_obj->is_objArray(), "must be obj array");
assert(from_obj->is_forwarded(), "must be forwarded");

oop to_obj = from_obj->forwardee();
@@ -248,7 +247,6 @@ MAYBE_INLINE_EVACUATION
void G1ParScanThreadState::start_partial_objarray(G1HeapRegionAttr dest_attr,
oop from_obj,
oop to_obj) {
assert(from_obj->is_objArray(), "precondition");
assert(from_obj->is_forwarded(), "precondition");
assert(from_obj->forwardee() == to_obj, "precondition");
assert(from_obj != to_obj, "should not be scanning self-forwarded objects");
@@ -428,9 +426,17 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
assert(region_attr.is_in_cset(),
"Unexpected region attr type: %s", region_attr.get_type_str());

if (old_mark.is_marked()) {
// Already forwarded by somebody else, return forwardee.
return old->forwardee(old_mark);
}
// Get the klass once. We'll need it again later, and this avoids
// re-decoding when it's compressed.
#ifdef _LP64
Klass* klass = old_mark.safe_klass();
#else
Klass* klass = old->klass();
#endif
const size_t word_sz = old->size_given_klass(klass);

uint age = 0;
@@ -83,7 +83,7 @@ void PSPromotionLAB::flush() {
// so they can always fill with an array.
HeapWord* tlab_end = end() + filler_header_size;
typeArrayOop filler_oop = (typeArrayOop) cast_to_oop(top());
filler_oop->set_mark(markWord::prototype());
filler_oop->set_mark(Universe::intArrayKlassObj()->prototype_header());
filler_oop->set_klass(Universe::intArrayKlassObj());
const size_t array_length =
pointer_delta(tlab_end, top()) - typeArrayOopDesc::header_size(T_INT);
@@ -314,7 +314,6 @@ void PSPromotionManager::process_array_chunk(PartialArrayScanTask task) {
assert(PSChunkLargeArrays, "invariant");

oop old = task.to_source_array();
assert(old->is_objArray(), "invariant");
assert(old->is_forwarded(), "invariant");

TASKQUEUE_STATS_ONLY(++_array_chunks_processed);
@@ -160,7 +160,12 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,

oop new_obj = NULL;
bool new_obj_is_tenured = false;
size_t new_obj_size = o->size();
#ifdef _LP64
Klass* klass = test_mark.safe_klass();
#else
Klass* klass = o->klass();
#endif
size_t new_obj_size = o->size_given_klass(klass);

// Find the objects age, MT safe.
uint age = (test_mark.has_displaced_mark_helper() /* o->has_displaced_mark() */) ?
@@ -41,7 +41,7 @@ inline void MarkSweep::mark_object(oop obj) {
// some marks may contain information we need to preserve so we store them away
// and overwrite the mark. We'll restore it at the end of markSweep.
markWord mark = obj->mark();
obj->set_mark(markWord::prototype().set_marked());
obj->set_mark(obj->klass()->prototype_header().set_marked());

if (obj->mark_must_be_preserved(mark)) {
preserve_mark(obj, mark);
@@ -208,10 +208,6 @@ bool CollectedHeap::is_oop(oop object) const {
return false;
}

if (is_in(object->klass_or_null())) {
return false;
}

return true;
}

@@ -48,7 +48,20 @@ inline void PreservedMarks::push_if_necessary(oop obj, markWord m) {
}

inline void PreservedMarks::init_forwarded_mark(oop obj) {
obj->init_mark();
assert(obj->is_forwarded(), "only forwarded here");
#ifdef _LP64
oop forwardee = obj->forwardee();
markWord header = forwardee->mark();
if (header.has_displaced_mark_helper()) {
header = header.displaced_mark_helper();
}
assert(UseCompressedClassPointers, "assume +UseCompressedClassPointers");
narrowKlass nklass = header.narrow_klass();
rkennke marked this conversation as resolved.
Show resolved Hide resolved
assert(nklass == obj->narrow_klass_legacy(), "narrow klass must match: header: " PTR_FORMAT ", nklass: " PTR_FORMAT, forwardee->mark().value(), uintptr_t(nklass));
obj->set_mark(markWord::prototype().set_narrow_klass(nklass));
#else
obj->set_mark(markWord::prototype());
#endif
}

inline PreservedMarks::PreservedMarks()
@@ -87,6 +87,9 @@ HeapWord* SlidingForwarding::decode_forwarding(HeapWord* original, uintptr_t enc
void SlidingForwarding::forward_to(oop original, oop target) {
#ifdef _LP64
markWord header = original->mark();
if (header.has_displaced_mark_helper()) {
header = header.displaced_mark_helper();
}
uintptr_t encoded = encode_forwarding(cast_from_oop<HeapWord*>(original), cast_from_oop<HeapWord*>(target));
assert((encoded & markWord::klass_mask_in_place) == 0, "encoded forwardee must not overlap with Klass*: " PTR_FORMAT, encoded);
header = markWord((header.value() & markWord::klass_mask_in_place) | encoded);
@@ -126,7 +126,7 @@ class InstanceRefKlass: public InstanceKlass {
static void oop_oop_iterate_fields_except_referent(oop obj, OopClosureType* closure, Contains& contains);

template <typename T>
static void trace_reference_gc(const char *s, oop obj) NOT_DEBUG_RETURN;
void trace_reference_gc(const char *s, oop obj) NOT_DEBUG_RETURN;

public:
// Update non-static oop maps so 'referent', 'nextPending' and
@@ -185,7 +185,7 @@ void InstanceRefKlass::trace_reference_gc(const char *s, oop obj) {
T* discovered_addr = (T*) java_lang_ref_Reference::discovered_addr_raw(obj);

log_develop_trace(gc, ref)("InstanceRefKlass %s for obj " PTR_FORMAT, s, p2i(obj));
if (java_lang_ref_Reference::is_phantom(obj)) {
if (reference_type() == REF_PHANTOM) {
log_develop_trace(gc, ref)(" referent_addr/* " PTR_FORMAT " / " PTR_FORMAT,
p2i(referent_addr), p2i((oop)HeapAccess<ON_PHANTOM_OOP_REF | AS_NO_KEEPALIVE>::oop_load(referent_addr)));
} else {
@@ -94,24 +94,3 @@ void markWord::print_on(outputStream* st, bool print_monitor_info) const {
st->print(" age=%d)", age());
}
}

#ifdef _LP64
narrowKlass markWord::narrow_klass() const {
return narrowKlass(value() >> klass_shift);
}

Klass* markWord::klass() const {
return CompressedKlassPointers::decode(narrow_klass());
}

markWord markWord::set_narrow_klass(const narrowKlass nklass) const {
return markWord((value() & ~klass_mask_in_place) | ((uintptr_t) nklass << klass_shift));
}

markWord markWord::set_klass(const Klass* klass) const {
assert(UseCompressedClassPointers, "expect compressed klass pointers");
// TODO: Don't cast to non-const, change CKP::encode() to accept const Klass* instead.
narrowKlass nklass = CompressedKlassPointers::encode(const_cast<Klass*>(klass));
return set_narrow_klass(nklass);
}
#endif
@@ -248,10 +248,12 @@ class markWord {
}

#ifdef _LP64
narrowKlass narrow_klass() const;
Klass* klass() const;
markWord set_klass(const Klass* klass) const;
markWord set_narrow_klass(const narrowKlass klass) const;
inline Klass* klass() const;
inline Klass* klass_or_null() const;
inline Klass* safe_klass() const;
inline markWord set_klass(const Klass* klass) const;
inline narrowKlass narrow_klass() const;
inline markWord set_narrow_klass(const narrowKlass klass) const;
#endif

// Prototype mark for initialization
@@ -0,0 +1,66 @@
/*
* Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/

#ifndef SHARE_OOPS_MARKWORD_INLINE_HPP
#define SHARE_OOPS_MARKWORD_INLINE_HPP

#include "oops/compressedOops.inline.hpp"
#include "oops/markWord.hpp"
#include "runtime/safepoint.hpp"

#ifdef _LP64
narrowKlass markWord::narrow_klass() const {
return narrowKlass(value() >> klass_shift);
}

Klass* markWord::klass() const {
return CompressedKlassPointers::decode_not_null(narrow_klass());
}

Klass* markWord::klass_or_null() const {
return CompressedKlassPointers::decode(narrow_klass());
}

markWord markWord::set_narrow_klass(const narrowKlass nklass) const {
return markWord((value() & ~klass_mask_in_place) | ((uintptr_t) nklass << klass_shift));
}

Klass* markWord::safe_klass() const {
assert(SafepointSynchronize::is_at_safepoint(), "only call at safepoint");
markWord m = *this;
if (m.has_displaced_mark_helper()) {
m = m.displaced_mark_helper();
}
return CompressedKlassPointers::decode_not_null(m.narrow_klass());
}

markWord markWord::set_klass(const Klass* klass) const {
assert(UseCompressedClassPointers, "expect compressed klass pointers");
// TODO: Don't cast to non-const, change CKP::encode() to accept const Klass* instead.
narrowKlass nklass = CompressedKlassPointers::encode(const_cast<Klass*>(klass));
return set_narrow_klass(nklass);
}
#endif

#endif // SHARE_OOPS_MARKWORD_INLINE_HPP
@@ -156,7 +156,6 @@ ObjArrayKlass::ObjArrayKlass(int n, Klass* element_klass, Symbol* name) : ArrayK
}

int ObjArrayKlass::oop_size(oop obj) const {
assert(obj->is_objArray(), "must be object array");
return objArrayOop(obj)->object_size();
}

@@ -70,7 +70,6 @@ void ObjArrayKlass::oop_oop_iterate_elements_bounded(

template <typename T, typename OopClosureType>
void ObjArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure) {
assert (obj->is_array(), "obj must be array");
objArrayOop a = objArrayOop(obj);

if (Devirtualizer::do_metadata(closure)) {
@@ -77,6 +77,7 @@ class oopDesc {
inline Klass* klass_or_null() const;
inline Klass* klass_or_null_acquire() const;

narrowKlass narrow_klass_legacy() const { return _metadata._compressed_klass; }
void set_narrow_klass(narrowKlass nk) NOT_CDS_JAVA_HEAP_RETURN;
inline void set_klass(Klass* k);
static inline void release_set_klass(HeapWord* mem, Klass* k);