Skip to content

Commit 69c3470

Browse files
committed
8252973: ZGC: Implement Large Pages support on Windows
Reviewed-by: eosterlund, mbeckwit, pliden
1 parent e4a32be commit 69c3470

11 files changed

+461
-157
lines changed

src/hotspot/os/posix/gc/z/zVirtualMemory_posix.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,11 @@
2929
#include <sys/mman.h>
3030
#include <sys/types.h>
3131

32-
void ZVirtualMemoryManager::pd_initialize() {
32+
void ZVirtualMemoryManager::pd_initialize_before_reserve() {
33+
// Does nothing
34+
}
35+
36+
void ZVirtualMemoryManager::pd_initialize_after_reserve() {
3337
// Does nothing
3438
}
3539

src/hotspot/os/windows/gc/z/zLargePages_windows.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,19 @@
2222
*/
2323

2424
#include "precompiled.hpp"
25+
#include "gc/shared/gcLogPrecious.hpp"
2526
#include "gc/z/zLargePages.hpp"
27+
#include "gc/z/zSyscall_windows.hpp"
28+
#include "runtime/globals.hpp"
2629

2730
void ZLargePages::pd_initialize() {
31+
if (UseLargePages) {
32+
if (ZSyscall::is_large_pages_supported()) {
33+
_state = Explicit;
34+
return;
35+
}
36+
log_info_p(gc, init)("Shared large pages not supported on this OS version");
37+
}
38+
2839
_state = Disabled;
2940
}

src/hotspot/os/windows/gc/z/zMapper_windows.cpp

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,62 @@ void ZMapper::close_paging_file_mapping(HANDLE file_handle) {
199199
}
200200
}
201201

202+
HANDLE ZMapper::create_shared_awe_section() {
203+
MEM_EXTENDED_PARAMETER parameter = { 0 };
204+
parameter.Type = MemSectionExtendedParameterUserPhysicalFlags;
205+
parameter.ULong64 = 0;
206+
207+
HANDLE section = ZSyscall::CreateFileMapping2(
208+
INVALID_HANDLE_VALUE, // File
209+
NULL, // SecurityAttributes
210+
SECTION_MAP_READ | SECTION_MAP_WRITE, // DesiredAccess
211+
PAGE_READWRITE, // PageProtection
212+
SEC_RESERVE | SEC_LARGE_PAGES, // AllocationAttributes
213+
0, // MaximumSize
214+
NULL, // Name
215+
&parameter, // ExtendedParameters
216+
1 // ParameterCount
217+
);
218+
219+
if (section == NULL) {
220+
fatal("Could not create shared AWE section (%d)", GetLastError());
221+
}
222+
223+
return section;
224+
}
225+
226+
uintptr_t ZMapper::reserve_for_shared_awe(HANDLE awe_section, uintptr_t addr, size_t size) {
227+
MEM_EXTENDED_PARAMETER parameter = { 0 };
228+
parameter.Type = MemExtendedParameterUserPhysicalHandle;
229+
parameter.Handle = awe_section;
230+
231+
void* const res = ZSyscall::VirtualAlloc2(
232+
GetCurrentProcess(), // Process
233+
(void*)addr, // BaseAddress
234+
size, // Size
235+
MEM_RESERVE | MEM_PHYSICAL, // AllocationType
236+
PAGE_READWRITE, // PageProtection
237+
&parameter, // ExtendedParameters
238+
1 // ParameterCount
239+
);
240+
241+
// Caller responsible for error handling
242+
return (uintptr_t)res;
243+
}
244+
245+
void ZMapper::unreserve_for_shared_awe(uintptr_t addr, size_t size) {
246+
bool res = VirtualFree(
247+
(void*)addr, // lpAddress
248+
0, // dwSize
249+
MEM_RELEASE // dwFreeType
250+
);
251+
252+
if (!res) {
253+
fatal("Failed to unreserve memory: " PTR_FORMAT " " SIZE_FORMAT "M (%d)",
254+
addr, size / M, GetLastError());
255+
}
256+
}
257+
202258
void ZMapper::split_placeholder(uintptr_t addr, size_t size) {
203259
const bool res = VirtualFree(
204260
(void*)addr, // lpAddress

src/hotspot/os/windows/gc/z/zMapper_windows.hpp

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,15 @@ class ZMapper : public AllStatic {
5959
// Close paging file mapping
6060
static void close_paging_file_mapping(HANDLE file_handle);
6161

62+
// Create a shared AWE section
63+
static HANDLE create_shared_awe_section();
64+
65+
// Reserve memory attached to the shared AWE section
66+
static uintptr_t reserve_for_shared_awe(HANDLE awe_section, uintptr_t addr, size_t size);
67+
68+
// Unreserve memory attached to a shared AWE section
69+
static void unreserve_for_shared_awe(uintptr_t addr, size_t size);
70+
6271
// Split a placeholder
6372
//
6473
// A view can only replace an entire placeholder, so placeholders need to be

src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.cpp

Lines changed: 179 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -24,98 +24,229 @@
2424
#include "precompiled.hpp"
2525
#include "gc/z/zGlobals.hpp"
2626
#include "gc/z/zGranuleMap.inline.hpp"
27+
#include "gc/z/zLargePages.inline.hpp"
2728
#include "gc/z/zMapper_windows.hpp"
2829
#include "gc/z/zPhysicalMemoryBacking_windows.hpp"
2930
#include "logging/log.hpp"
3031
#include "runtime/globals.hpp"
3132
#include "utilities/debug.hpp"
3233

34+
class ZPhysicalMemoryBackingImpl : public CHeapObj<mtGC> {
35+
public:
36+
virtual size_t commit(size_t offset, size_t size) = 0;
37+
virtual size_t uncommit(size_t offset, size_t size) = 0;
38+
virtual void map(uintptr_t addr, size_t size, size_t offset) const = 0;
39+
virtual void unmap(uintptr_t addr, size_t size) const = 0;
40+
};
41+
42+
// Implements small pages (paged) support using placeholder reservation.
43+
//
3344
// The backing commits and uncommits physical memory, that can be
3445
// multi-mapped into the virtual address space. To support fine-graned
3546
// committing and uncommitting, each ZGranuleSize'd chunk is mapped to
3647
// a separate paging file mapping.
3748

38-
ZPhysicalMemoryBacking::ZPhysicalMemoryBacking(size_t max_capacity) :
39-
_handles(max_capacity) {}
49+
class ZPhysicalMemoryBackingSmallPages : public ZPhysicalMemoryBackingImpl {
50+
private:
51+
ZGranuleMap<HANDLE> _handles;
4052

41-
bool ZPhysicalMemoryBacking::is_initialized() const {
42-
return true;
43-
}
53+
HANDLE get_handle(uintptr_t offset) const {
54+
HANDLE const handle = _handles.get(offset);
55+
assert(handle != 0, "Should be set");
56+
return handle;
57+
}
4458

45-
void ZPhysicalMemoryBacking::warn_commit_limits(size_t max_capacity) const {
46-
// Does nothing
47-
}
59+
void put_handle(uintptr_t offset, HANDLE handle) {
60+
assert(handle != INVALID_HANDLE_VALUE, "Invalid handle");
61+
assert(_handles.get(offset) == 0, "Should be cleared");
62+
_handles.put(offset, handle);
63+
}
4864

49-
HANDLE ZPhysicalMemoryBacking::get_handle(uintptr_t offset) const {
50-
HANDLE const handle = _handles.get(offset);
51-
assert(handle != 0, "Should be set");
52-
return handle;
53-
}
65+
void clear_handle(uintptr_t offset) {
66+
assert(_handles.get(offset) != 0, "Should be set");
67+
_handles.put(offset, 0);
68+
}
5469

55-
void ZPhysicalMemoryBacking::put_handle(uintptr_t offset, HANDLE handle) {
56-
assert(handle != INVALID_HANDLE_VALUE, "Invalid handle");
57-
assert(_handles.get(offset) == 0, "Should be cleared");
58-
_handles.put(offset, handle);
59-
}
70+
public:
71+
ZPhysicalMemoryBackingSmallPages(size_t max_capacity) :
72+
ZPhysicalMemoryBackingImpl(),
73+
_handles(max_capacity) {}
6074

61-
void ZPhysicalMemoryBacking::clear_handle(uintptr_t offset) {
62-
assert(_handles.get(offset) != 0, "Should be set");
63-
_handles.put(offset, 0);
64-
}
75+
size_t commit(size_t offset, size_t size) {
76+
for (size_t i = 0; i < size; i += ZGranuleSize) {
77+
HANDLE const handle = ZMapper::create_and_commit_paging_file_mapping(ZGranuleSize);
78+
if (handle == 0) {
79+
return i;
80+
}
6581

66-
size_t ZPhysicalMemoryBacking::commit_from_paging_file(size_t offset, size_t size) {
67-
for (size_t i = 0; i < size; i += ZGranuleSize) {
68-
HANDLE const handle = ZMapper::create_and_commit_paging_file_mapping(ZGranuleSize);
69-
if (handle == 0) {
70-
return i;
82+
put_handle(offset + i, handle);
7183
}
7284

73-
put_handle(offset + i, handle);
85+
return size;
7486
}
7587

76-
return size;
77-
}
88+
size_t uncommit(size_t offset, size_t size) {
89+
for (size_t i = 0; i < size; i += ZGranuleSize) {
90+
HANDLE const handle = get_handle(offset + i);
91+
clear_handle(offset + i);
92+
ZMapper::close_paging_file_mapping(handle);
93+
}
94+
95+
return size;
96+
}
97+
98+
void map(uintptr_t addr, size_t size, size_t offset) const {
99+
assert(is_aligned(offset, ZGranuleSize), "Misaligned");
100+
assert(is_aligned(addr, ZGranuleSize), "Misaligned");
101+
assert(is_aligned(size, ZGranuleSize), "Misaligned");
102+
103+
for (size_t i = 0; i < size; i += ZGranuleSize) {
104+
HANDLE const handle = get_handle(offset + i);
105+
ZMapper::map_view_replace_placeholder(handle, 0 /* offset */, addr + i, ZGranuleSize);
106+
}
107+
}
108+
109+
void unmap(uintptr_t addr, size_t size) const {
110+
assert(is_aligned(addr, ZGranuleSize), "Misaligned");
111+
assert(is_aligned(size, ZGranuleSize), "Misaligned");
112+
113+
for (size_t i = 0; i < size; i += ZGranuleSize) {
114+
ZMapper::unmap_view_preserve_placeholder(addr + i, ZGranuleSize);
115+
}
116+
}
117+
};
118+
119+
// Implements Large Pages (locked) support using shared AWE physical memory.
120+
//
121+
// Shared AWE physical memory also works with small pages, but it has
122+
// a few drawbacks that makes it a no-go to use it at this point:
123+
//
124+
// 1) It seems to use 8 bytes of committed memory per *reserved* memory.
125+
// Given our scheme to use a large address space range this turns out to
126+
// use too much memory.
127+
//
128+
// 2) It requires memory locking privilages, even for small pages. This
129+
// has always been a requirement for large pages, and would be an extra
130+
// restriction for usage with small pages.
131+
//
132+
// Note: The large pages size is tied to our ZGranuleSize.
133+
134+
extern HANDLE ZAWESection;
135+
136+
class ZPhysicalMemoryBackingLargePages : public ZPhysicalMemoryBackingImpl {
137+
private:
138+
ULONG_PTR* const _page_array;
139+
140+
static ULONG_PTR* alloc_page_array(size_t max_capacity) {
141+
const size_t npages = max_capacity / ZGranuleSize;
142+
const size_t array_size = npages * sizeof(ULONG_PTR);
143+
144+
return (ULONG_PTR*)os::malloc(array_size, mtGC);
145+
}
146+
147+
public:
148+
ZPhysicalMemoryBackingLargePages(size_t max_capacity) :
149+
ZPhysicalMemoryBackingImpl(),
150+
_page_array(alloc_page_array(max_capacity)) {}
151+
152+
size_t commit(size_t offset, size_t size) {
153+
const size_t index = offset >> ZGranuleSizeShift;
154+
const size_t npages = size >> ZGranuleSizeShift;
155+
156+
size_t npages_res = npages;
157+
const bool res = AllocateUserPhysicalPages(ZAWESection, &npages_res, &_page_array[index]);
158+
if (!res) {
159+
fatal("Failed to allocate physical memory " SIZE_FORMAT "M @ " PTR_FORMAT " (%d)",
160+
size / M, offset, GetLastError());
161+
} else {
162+
log_debug(gc)("Allocated physical memory: " SIZE_FORMAT "M @ " PTR_FORMAT, size / M, offset);
163+
}
78164

79-
size_t ZPhysicalMemoryBacking::uncommit_from_paging_file(size_t offset, size_t size) {
80-
for (size_t i = 0; i < size; i += ZGranuleSize) {
81-
HANDLE const handle = get_handle(offset + i);
82-
clear_handle(offset + i);
83-
ZMapper::close_paging_file_mapping(handle);
165+
// AllocateUserPhysicalPages might not be able to allocate the requested amount of memory.
166+
// The allocated number of pages are written in npages_res.
167+
return npages_res << ZGranuleSizeShift;
84168
}
85169

86-
return size;
170+
size_t uncommit(size_t offset, size_t size) {
171+
const size_t index = offset >> ZGranuleSizeShift;
172+
const size_t npages = size >> ZGranuleSizeShift;
173+
174+
size_t npages_res = npages;
175+
const bool res = FreeUserPhysicalPages(ZAWESection, &npages_res, &_page_array[index]);
176+
if (!res) {
177+
fatal("Failed to uncommit physical memory " SIZE_FORMAT "M @ " PTR_FORMAT " (%d)",
178+
size, offset, GetLastError());
179+
}
180+
181+
return npages_res << ZGranuleSizeShift;
182+
}
183+
184+
void map(uintptr_t addr, size_t size, size_t offset) const {
185+
const size_t npages = size >> ZGranuleSizeShift;
186+
const size_t index = offset >> ZGranuleSizeShift;
187+
188+
const bool res = MapUserPhysicalPages((char*)addr, npages, &_page_array[index]);
189+
if (!res) {
190+
fatal("Failed to map view " PTR_FORMAT " " SIZE_FORMAT "M @ " PTR_FORMAT " (%d)",
191+
addr, size / M, offset, GetLastError());
192+
}
193+
}
194+
195+
void unmap(uintptr_t addr, size_t size) const {
196+
const size_t npages = size >> ZGranuleSizeShift;
197+
198+
const bool res = MapUserPhysicalPages((char*)addr, npages, NULL);
199+
if (!res) {
200+
fatal("Failed to unmap view " PTR_FORMAT " " SIZE_FORMAT "M (%d)",
201+
addr, size / M, GetLastError());
202+
}
203+
}
204+
};
205+
206+
static ZPhysicalMemoryBackingImpl* select_impl(size_t max_capacity) {
207+
if (ZLargePages::is_enabled()) {
208+
return new ZPhysicalMemoryBackingLargePages(max_capacity);
209+
}
210+
211+
return new ZPhysicalMemoryBackingSmallPages(max_capacity);
212+
}
213+
214+
ZPhysicalMemoryBacking::ZPhysicalMemoryBacking(size_t max_capacity) :
215+
_impl(select_impl(max_capacity)) {}
216+
217+
bool ZPhysicalMemoryBacking::is_initialized() const {
218+
return true;
219+
}
220+
221+
void ZPhysicalMemoryBacking::warn_commit_limits(size_t max_capacity) const {
222+
// Does nothing
87223
}
88224

89225
size_t ZPhysicalMemoryBacking::commit(size_t offset, size_t length) {
90226
log_trace(gc, heap)("Committing memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
91227
offset / M, (offset + length) / M, length / M);
92228

93-
return commit_from_paging_file(offset, length);
229+
return _impl->commit(offset, length);
94230
}
95231

96232
size_t ZPhysicalMemoryBacking::uncommit(size_t offset, size_t length) {
97233
log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
98234
offset / M, (offset + length) / M, length / M);
99235

100-
return uncommit_from_paging_file(offset, length);
236+
return _impl->uncommit(offset, length);
101237
}
102238

103239
void ZPhysicalMemoryBacking::map(uintptr_t addr, size_t size, size_t offset) const {
104-
assert(is_aligned(offset, ZGranuleSize), "Misaligned");
105-
assert(is_aligned(addr, ZGranuleSize), "Misaligned");
106-
assert(is_aligned(size, ZGranuleSize), "Misaligned");
240+
assert(is_aligned(offset, ZGranuleSize), "Misaligned: " PTR_FORMAT, offset);
241+
assert(is_aligned(addr, ZGranuleSize), "Misaligned: " PTR_FORMAT, addr);
242+
assert(is_aligned(size, ZGranuleSize), "Misaligned: " PTR_FORMAT, size);
107243

108-
for (size_t i = 0; i < size; i += ZGranuleSize) {
109-
HANDLE const handle = get_handle(offset + i);
110-
ZMapper::map_view_replace_placeholder(handle, 0 /* offset */, addr + i, ZGranuleSize);
111-
}
244+
_impl->map(addr, size, offset);
112245
}
113246

114247
void ZPhysicalMemoryBacking::unmap(uintptr_t addr, size_t size) const {
115248
assert(is_aligned(addr, ZGranuleSize), "Misaligned");
116249
assert(is_aligned(size, ZGranuleSize), "Misaligned");
117250

118-
for (size_t i = 0; i < size; i += ZGranuleSize) {
119-
ZMapper::unmap_view_preserve_placeholder(addr + i, ZGranuleSize);
120-
}
251+
_impl->unmap(addr, size);
121252
}

0 commit comments

Comments
 (0)