Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

8256155: Allow multiple large page sizes to be used on Linux #1153

Closed
wants to merge 61 commits into from
Closed
Changes from all commits
Commits
Show all changes
61 commits
Select commit Hold shift + click to select a range
7092bec
Add 2M LargePages to _page_sizes
mgkwill Oct 15, 2020
5f78b27
Add 2M LargePages to _page_sizes
mgkwill Oct 15, 2020
a2f6f8f
Merge remote-tracking branch 'upstream/master' into update_hlp
mgkwill Dec 1, 2020
57e5496
Merge branch 'update_hlp' of github.com:mgkwill/jdk into update_hlp
mgkwill Dec 1, 2020
eb9e9ea
Merge branch 'master' into update_hlp
mgkwill Dec 2, 2020
979742c
JDK-8257588: Make os::_page_sizes a bitmask #1522
mgkwill Dec 3, 2020
0901e70
Adress Comments, Rework changes for PagesizeSet
mgkwill Dec 3, 2020
b5bd144
Remove remnant UseSHM change
mgkwill Dec 4, 2020
4e66e85
Merge branch 'master' into update_hlp
mgkwill Dec 7, 2020
1a33482
Thomas S. Feedback
mgkwill Dec 7, 2020
5cd6d6a
Fix merge mistakes
mgkwill Dec 7, 2020
86dc577
Fix merge mistakes
mgkwill Dec 7, 2020
b7a0d33
Add newline at end of TestLargePageUseForAuxMemory.java
mgkwill Dec 7, 2020
073ffab
Merge branch 'update_hlp' of github.com:mgkwill/jdk into update_hlp
mgkwill Dec 7, 2020
870e8a5
Fix space format, use Linux:: for local func.
mgkwill Dec 8, 2020
0bfc0cb
Ivan W. Requested Changes
mgkwill Dec 8, 2020
85e7502
Fix os::large_page_size() in last update
mgkwill Dec 8, 2020
70bd901
Merge branch 'master' into update_hlp
mgkwill Dec 8, 2020
8f1474a
Merge branch 'master' into update_hlp
mgkwill Dec 9, 2020
7aaa093
Merge branch 'master' into update_hlp
mgkwill Dec 9, 2020
364b2b5
Remove extraneous ' from warning
mgkwill Dec 10, 2020
81ff7c5
Merge branch 'master' into update_hlp
mgkwill Dec 10, 2020
d73e7a4
Merge branch 'master' into update_hlp
mgkwill Dec 15, 2020
c13bc84
kstefanj update
mgkwill Feb 11, 2021
f2e44ac
Merge branch 'master' into pull/1153
mgkwill Feb 16, 2021
30e01e1
Cast os::vm_page_size to size_t, fix build
mgkwill Mar 3, 2021
03ab5e5
Merge remote-tracking branch 'upstream/master' into update_hlp
mgkwill Mar 10, 2021
e0c5461
Update LargePage Setup per review comments
mgkwill Mar 10, 2021
4ed6b44
Fix first set of TestTracePageSizes.java issues
mgkwill Mar 11, 2021
90befbe
Fix whitespace error
mgkwill Mar 11, 2021
f3b4b81
Fix reserve_memory_special_huge_tlbfs_mixed, remove logging
mgkwill Mar 15, 2021
3cfeb95
Fix logging issues
mgkwill Mar 15, 2021
6a8f57f
Fix logging issues 2
mgkwill Mar 15, 2021
22b27b9
Use SIZE_FORMAT in logging
mgkwill Mar 15, 2021
1ecb4e6
Addressed kstefanj review suggestions
mgkwill Mar 16, 2021
08bf288
Merge branch 'master' into update_hlp
mgkwill Mar 16, 2021
6faf7e1
8262291: Refactor reserve_memory_special_huge_tlbfs
kstefanj Feb 25, 2021
38a1314
Ivan review
kstefanj Mar 24, 2021
2bf1d5a
Marcus review.
kstefanj Mar 24, 2021
f70ca6a
Self review.
kstefanj Mar 25, 2021
787b87f
Thomas review.
kstefanj Mar 26, 2021
5110016
Merge branch 'pull/3073' into update_hlp
mgkwill Mar 26, 2021
a9b3dee
Rebase on pull/3073
mgkwill Mar 29, 2021
f5bfe22
Merge branch 'master' into update_hlp
mgkwill Apr 2, 2021
064fa64
Merge branch 'master' into update_hlp
mgkwill Apr 20, 2021
f99fd5d
Fix merge
mgkwill Apr 20, 2021
f3abe25
Merge branch 'master' into update_hlp
mgkwill Apr 23, 2021
6f06330
Set LargePageSizeInBytes to largepage upper limit
mgkwill Apr 23, 2021
6cd6be1
kstefanj review
mgkwill Apr 28, 2021
a2495c4
Remove extranous vm_page_size check
mgkwill May 5, 2021
006de8a
Merge branch 'master' into update_hlp
mgkwill May 7, 2021
be8f95a
Merge branch 'master' into update_hlp
mgkwill May 7, 2021
7368e7d
Remove reserve_memory_special_huge_tlbfs mods except assert
mgkwill May 7, 2021
024cbc5
Review comments 5_12
mgkwill May 12, 2021
7a3c1e4
Check large pages =< _large_page_size
mgkwill May 12, 2021
df16a00
Merge branch 'master' into update_hlp
mgkwill May 12, 2021
a551a5e
Check smaller LP sizes in setup
mgkwill May 12, 2021
6a00a18
Reuse flags & p vars, fix enclose
mgkwill May 12, 2021
581346a
kstefanj review comments
mgkwill May 17, 2021
aa68677
Merge branch 'master' into update_hlp
mgkwill May 20, 2021
1a487de
tstuefe review
mgkwill May 20, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
@@ -2346,6 +2346,9 @@ void os::print_memory_info(outputStream* st) {
st->print("(" UINT64_FORMAT "k free)",
((jlong)si.freeswap * si.mem_unit) >> 10);
st->cr();
st->print("Page Sizes: ");
_page_sizes.print_on(st);
st->cr();
}

// Print the first "model name" line and the first "flags" line
@@ -3504,6 +3507,25 @@ bool os::Linux::hugetlbfs_sanity_check(bool warn, size_t page_size) {
// Mapping succeeded, sanity check passed.
munmap(p, page_size);
return true;
} else {
log_info(pagesize)("Large page size (" SIZE_FORMAT "%s) failed sanity check, "
"checking if smaller large page sizes are usable",
byte_size_in_exact_unit(page_size),
exact_unit_for_byte_size(page_size));
for (size_t page_size_ = _page_sizes.next_smaller(page_size);
page_size_ != (size_t)os::vm_page_size();
page_size_ = _page_sizes.next_smaller(page_size_)) {
flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_HUGETLB | hugetlbfs_page_size_flag(page_size_);
p = mmap(NULL, page_size_, PROT_READ|PROT_WRITE, flags, -1, 0);
if (p != MAP_FAILED) {
// Mapping succeeded, sanity check passed.
munmap(p, page_size_);
log_info(pagesize)("Large page size (" SIZE_FORMAT "%s) passed sanity check",
byte_size_in_exact_unit(page_size_),
exact_unit_for_byte_size(page_size_));
return true;
}
}
}

if (warn) {
@@ -3578,35 +3600,20 @@ static void set_coredump_filter(CoredumpFilterBit bit) {

static size_t _large_page_size = 0;

size_t os::Linux::find_default_large_page_size() {
if (_default_large_page_size != 0) {
return _default_large_page_size;
}
size_t large_page_size = 0;
static size_t scan_default_large_page_size() {
size_t default_large_page_size = 0;

// large_page_size on Linux is used to round up heap size. x86 uses either
// 2M or 4M page, depending on whether PAE (Physical Address Extensions)
// mode is enabled. AMD64/EM64T uses 2M page in 64bit mode. IA64 can use
// page as large as 256M.
// page as large as 1G.
//
// Here we try to figure out page size by parsing /proc/meminfo and looking
// for a line with the following format:
// Hugepagesize: 2048 kB
//
// If we can't determine the value (e.g. /proc is not mounted, or the text
// format has been changed), we'll use the largest page size supported by
// the processor.

#ifndef ZERO
large_page_size =
AARCH64_ONLY(2 * M)
AMD64_ONLY(2 * M)
ARM32_ONLY(2 * M)
IA32_ONLY(4 * M)
IA64_ONLY(256 * M)
PPC_ONLY(4 * M)
S390_ONLY(1 * M);
#endif // ZERO
mgkwill marked this conversation as resolved.
Show resolved Hide resolved
// format has been changed), we'll set largest page size to 0
mgkwill marked this conversation as resolved.
Show resolved Hide resolved

FILE *fp = fopen("/proc/meminfo", "r");
if (fp) {
@@ -3615,7 +3622,7 @@ size_t os::Linux::find_default_large_page_size() {
char buf[16];
if (fscanf(fp, "Hugepagesize: %d", &x) == 1) {
if (x && fgets(buf, sizeof(buf), fp) && strcmp(buf, " kB\n") == 0) {
large_page_size = x * K;
default_large_page_size = x * K;
break;
}
} else {
@@ -3628,65 +3635,50 @@ size_t os::Linux::find_default_large_page_size() {
}
fclose(fp);
}
return large_page_size;

return default_large_page_size;
}

size_t os::Linux::find_large_page_size(size_t large_page_size) {
if (_default_large_page_size == 0) {
_default_large_page_size = Linux::find_default_large_page_size();
}
// We need to scan /sys/kernel/mm/hugepages
static os::PageSizes scan_multiple_page_support() {
// Scan /sys/kernel/mm/hugepages
// to discover the available page sizes
const char* sys_hugepages = "/sys/kernel/mm/hugepages";
os::PageSizes page_sizes;

DIR *dir = opendir(sys_hugepages);
if (dir == NULL) {
return _default_large_page_size;
}

struct dirent *entry;
size_t page_size;
while ((entry = readdir(dir)) != NULL) {
if (entry->d_type == DT_DIR &&
sscanf(entry->d_name, "hugepages-%zukB", &page_size) == 1) {
// The kernel is using kB, hotspot uses bytes
if (large_page_size == page_size * K) {
closedir(dir);
return large_page_size;
}
// Add each found Large Page Size to page_sizes
page_sizes.add(page_size * K);
}
}
closedir(dir);
mgkwill marked this conversation as resolved.
Show resolved Hide resolved
return _default_large_page_size;
}

size_t os::Linux::setup_large_page_size() {
_default_large_page_size = Linux::find_default_large_page_size();

if (!FLAG_IS_DEFAULT(LargePageSizeInBytes) && LargePageSizeInBytes != _default_large_page_size ) {
_large_page_size = find_large_page_size(LargePageSizeInBytes);
if (_large_page_size == _default_large_page_size) {
warning("Setting LargePageSizeInBytes=" SIZE_FORMAT " has no effect on this OS. Using the default large page size "
SIZE_FORMAT "%s.",
LargePageSizeInBytes,
byte_size_in_proper_unit(_large_page_size), proper_unit_for_byte_size(_large_page_size));
}
} else {
_large_page_size = _default_large_page_size;
}

const size_t default_page_size = (size_t)Linux::page_size();
if (_large_page_size > default_page_size) {
_page_sizes.add(_large_page_size);
LogTarget(Debug, pagesize) lt;
if (lt.is_enabled()) {
LogStream ls(lt);
ls.print("Large Page sizes: ");
page_sizes.print_on(&ls);
}
mgkwill marked this conversation as resolved.
Show resolved Hide resolved

return _large_page_size;
return page_sizes;
}

size_t os::Linux::default_large_page_size() {
return _default_large_page_size;
}

void warn_no_large_pages_configured() {
if (!FLAG_IS_DEFAULT(UseLargePages)) {
log_warning(pagesize)("UseLargePages disabled, no large pages configured and available on the system.");
}
}

bool os::Linux::setup_large_page_type(size_t page_size) {
if (FLAG_IS_DEFAULT(UseHugeTLBFS) &&
FLAG_IS_DEFAULT(UseSHM) &&
@@ -3729,13 +3721,13 @@ bool os::Linux::setup_large_page_type(size_t page_size) {
UseSHM = false;
}

if (!FLAG_IS_DEFAULT(UseLargePages)) {
log_warning(pagesize)("UseLargePages disabled, no large pages configured and available on the system.");
}
warn_no_large_pages_configured();
return false;
}

void os::large_page_init() {
// 1) Handle the case where we do not want to use huge pages and hence
// there is no need to scan the OS for related info
if (!UseLargePages &&
!UseTransparentHugePages &&
!UseHugeTLBFS &&
@@ -3753,8 +3745,73 @@ void os::large_page_init() {
return;
}

size_t large_page_size = Linux::setup_large_page_size();
UseLargePages = Linux::setup_large_page_type(large_page_size);
// 2) Scan OS info
size_t default_large_page_size = scan_default_large_page_size();
os::Linux::_default_large_page_size = default_large_page_size;
if (default_large_page_size == 0) {
// No large pages configured, return.
warn_no_large_pages_configured();
UseLargePages = false;
UseTransparentHugePages = false;
UseHugeTLBFS = false;
UseSHM = false;
mgkwill marked this conversation as resolved.
Show resolved Hide resolved
return;
}
os::PageSizes all_large_pages = scan_multiple_page_support();

// 3) Consistency check and post-processing

// It is unclear if /sys/kernel/mm/hugepages/ and /proc/meminfo could disagree. Manually
// re-add the default page size to the list of page sizes to be sure.
all_large_pages.add(default_large_page_size);

// Check LargePageSizeInBytes matches an available page size and if so set _large_page_size
// using LargePageSizeInBytes as the maximum allowed large page size. If LargePageSizeInBytes
// doesn't match an available page size set _large_page_size to default_large_page_size
// and use it as the maximum.
if (FLAG_IS_DEFAULT(LargePageSizeInBytes) ||
LargePageSizeInBytes == 0 ||
LargePageSizeInBytes == default_large_page_size) {
_large_page_size = default_large_page_size;
log_info(pagesize)("Using the default large page size: " SIZE_FORMAT "%s",
byte_size_in_exact_unit(_large_page_size),
exact_unit_for_byte_size(_large_page_size));
} else {
if (all_large_pages.contains(LargePageSizeInBytes)) {
_large_page_size = LargePageSizeInBytes;
log_info(pagesize)("Overriding default large page size (" SIZE_FORMAT "%s) "
"using LargePageSizeInBytes: " SIZE_FORMAT "%s",
byte_size_in_exact_unit(default_large_page_size),
exact_unit_for_byte_size(default_large_page_size),
byte_size_in_exact_unit(_large_page_size),
exact_unit_for_byte_size(_large_page_size));
} else {
_large_page_size = default_large_page_size;
log_info(pagesize)("LargePageSizeInBytes is not a valid large page size (" SIZE_FORMAT "%s) "
"using the default large page size: " SIZE_FORMAT "%s",
byte_size_in_exact_unit(LargePageSizeInBytes),
exact_unit_for_byte_size(LargePageSizeInBytes),
byte_size_in_exact_unit(_large_page_size),
exact_unit_for_byte_size(_large_page_size));
}
}

// Populate _page_sizes with large page sizes less than or equal to
// _large_page_size.
for (size_t page_size = _large_page_size; page_size != 0;
page_size = all_large_pages.next_smaller(page_size)) {
_page_sizes.add(page_size);
}

LogTarget(Info, pagesize) lt;
if (lt.is_enabled()) {
LogStream ls(lt);
ls.print("Usable page sizes: ");
_page_sizes.print_on(&ls);
mgkwill marked this conversation as resolved.
Show resolved Hide resolved
}

// Now determine the type of large pages to use:
UseLargePages = os::Linux::setup_large_page_type(_large_page_size);

set_coredump_filter(LARGEPAGES_BIT);
}
@@ -3965,6 +4022,7 @@ char* os::Linux::reserve_memory_special_huge_tlbfs(size_t bytes,
assert(is_aligned(req_addr, page_size), "Must be");
assert(is_aligned(alignment, os::vm_allocation_granularity()), "Must be");
assert(_page_sizes.contains(page_size), "Must be a valid page size");
assert(page_size > (size_t)os::vm_page_size(), "Must be a large page size");
assert(bytes >= page_size, "Shouldn't allocate large pages for small sizes");

// We only end up here when at least 1 large page can be used.
@@ -78,9 +78,8 @@ class Linux {
static GrowableArray<int>* nindex_to_node() { return _nindex_to_node; }

static size_t default_large_page_size();
static size_t find_default_large_page_size();
static size_t find_large_page_size(size_t page_size);
static size_t setup_large_page_size();
static size_t scan_default_large_page_size();
static os::PageSizes scan_multiple_page_support();

static bool setup_large_page_type(size_t page_size);
static bool transparent_huge_pages_sanity_check(bool warn, size_t pages_size);
@@ -239,7 +239,8 @@ const intx ObjectAlignmentInBytes = 8;
"Use intrinsics for java.util.Base64") \
\
product(size_t, LargePageSizeInBytes, 0, \
"Large page size (0 to let VM choose the page size)") \
"Maximum large page size used (0 will use the default large " \
"page size for the environment as the maximum)") \
range(0, max_uintx) \
\
product(size_t, LargePageHeapSizeThreshold, 128*M, \