2222 */
2323
2424#include " precompiled.hpp"
25+ #include " gc/z/zAddress.inline.hpp"
2526#include " gc/z/zForwarding.inline.hpp"
26- #include " utilities/debug.hpp"
27+ #include " gc/z/zStat.hpp"
28+ #include " gc/z/zUtils.inline.hpp"
29+ #include " utilities/align.hpp"
30+
31+ //
32+ // Reference count states:
33+ //
34+ // * If the reference count is zero, it will never change again.
35+ //
36+ // * If the reference count is positive, it can be both retained
37+ // (increased) and released (decreased).
38+ //
39+ // * If the reference count is negative, is can only be released
40+ // (increased). A negative reference count means that one or more
41+ // threads are waiting for one or more other threads to release
42+ // their references.
43+ //
44+ // The reference lock is used for waiting until the reference
45+ // count has become zero (released) or negative one (claimed).
46+ //
47+
48+ static const ZStatCriticalPhase ZCriticalPhaseRelocationStall (" Relocation Stall"
49+
50+ bool ZForwarding::retain_page () {
51+ for (;;) {
52+ const int32_t ref_count = Atomic::load_acquire (&_ref_count);
53+
54+ if (ref_count == 0 ) {
55+ // Released
56+ return false ;
57+ }
58+
59+ if (ref_count < 0 ) {
60+ // Claimed
61+ wait_page_released ();
62+ return false ;
63+ }
64+
65+ if (Atomic::cmpxchg (&_ref_count, ref_count, ref_count + 1 ) == ref_count) {
66+ // Retained
67+ return true ;
68+ }
69+ }
70+ }
71+
72+ ZPage* ZForwarding::claim_page () {
73+ for (;;) {
74+ const int32_t ref_count = Atomic::load (&_ref_count);
75+ assert (ref_count > 0 , " Invalid state"
76+
77+ // Invert reference count
78+ if (Atomic::cmpxchg (&_ref_count, ref_count, -ref_count) != ref_count) {
79+ continue ;
80+ }
81+
82+ // If the previous reference count was 1, then we just changed it to -1,
83+ // and we have now claimed the page. Otherwise we wait until it is claimed.
84+ if (ref_count != 1 ) {
85+ ZLocker<ZConditionLock> locker (&_ref_lock);
86+ while (Atomic::load_acquire (&_ref_count) != -1 ) {
87+ _ref_lock.wait ();
88+ }
89+ }
90+
91+ return _page;
92+ }
93+ }
94+
95+ void ZForwarding::release_page () {
96+ for (;;) {
97+ const int32_t ref_count = Atomic::load (&_ref_count);
98+ assert (ref_count != 0 , " Invalid state"
99+
100+ if (ref_count > 0 ) {
101+ // Decrement reference count
102+ if (Atomic::cmpxchg (&_ref_count, ref_count, ref_count - 1 ) != ref_count) {
103+ continue ;
104+ }
105+
106+ // If the previous reference count was 1, then we just decremented
107+ // it to 0 and we should signal that the page is now released.
108+ if (ref_count == 1 ) {
109+ // Notify released
110+ ZLocker<ZConditionLock> locker (&_ref_lock);
111+ _ref_lock.notify_all ();
112+ }
113+ } else {
114+ // Increment reference count
115+ if (Atomic::cmpxchg (&_ref_count, ref_count, ref_count + 1 ) != ref_count) {
116+ continue ;
117+ }
118+
119+ // If the previous reference count was -2 or -1, then we just incremented it
120+ // to -1 or 0, and we should signal the that page is now claimed or released.
121+ if (ref_count == -2 || ref_count == -1 ) {
122+ // Notify claimed or released
123+ ZLocker<ZConditionLock> locker (&_ref_lock);
124+ _ref_lock.notify_all ();
125+ }
126+ }
127+
128+ return ;
129+ }
130+ }
131+
132+ void ZForwarding::wait_page_released () const {
133+ if (Atomic::load_acquire (&_ref_count) != 0 ) {
134+ ZStatTimer timer (ZCriticalPhaseRelocationStall);
135+ ZLocker<ZConditionLock> locker (&_ref_lock);
136+ while (Atomic::load_acquire (&_ref_count) != 0 ) {
137+ _ref_lock.wait ();
138+ }
139+ }
140+ }
141+
142+ ZPage* ZForwarding::detach_page () {
143+ // Wait until released
144+ if (Atomic::load_acquire (&_ref_count) != 0 ) {
145+ ZLocker<ZConditionLock> locker (&_ref_lock);
146+ while (Atomic::load_acquire (&_ref_count) != 0 ) {
147+ _ref_lock.wait ();
148+ }
149+ }
150+
151+ // Detach and return page
152+ ZPage* const page = _page;
153+ _page = NULL ;
154+ return page;
155+ }
27156
28157void ZForwarding::verify () const {
29- guarantee (_refcount > 0 , " Invalid refcount "
158+ guarantee (_ref_count != 0 , " Invalid reference count "
30159 guarantee (_page != NULL , " Invalid page"
31160
32- size_t live_objects = 0 ;
161+ uint32_t live_objects = 0 ;
162+ size_t live_bytes = 0 ;
33163
34164 for (ZForwardingCursor i = 0 ; i < _entries.length (); i++) {
35165 const ZForwardingEntry entry = at (&i);
@@ -53,9 +183,13 @@ void ZForwarding::verify() const {
53183 guarantee (entry.to_offset () != other.to_offset (), " Duplicate to"
54184 }
55185
186+ const uintptr_t to_addr = ZAddress::good (entry.to_offset ());
187+ const size_t size = ZUtils::object_size (to_addr);
188+ const size_t aligned_size = align_up (size, _page->object_alignment ());
189+ live_bytes += aligned_size;
56190 live_objects++;
57191 }
58192
59- // Check number of non-empty entries
60- guarantee (live_objects == _page->live_objects (), " Invalid number of entries "
193+ // Verify number of live objects and bytes
194+ _page->verify_live (live_objects, live_bytes );
61195}
0 commit comments