Skip to content

Commit b05ba93

Browse files
committed
[Memory] Add basic support for large/huge memory pages
This patch introduces Memory::MF_HUGE_HINT which indicates that allocateMappedMemory() shall return a pointer to a large memory page. However the flag is a hint because we're not guaranteed in any way that we will get back a large memory page. There are several restrictions: - Large/huge memory pages aren't enabled by default on modern OSes (Windows 10 and Linux at least), and should be manually enabled/reserved. - Once enabled, it should be kept in mind that large pages are physical only, they can't be swapped. - Memory fragmentation can affect the availability of large pages, especially after running the OS for a long time and/or running along many other applications. Memory::allocateMappedMemory() will fallback to 4KB pages if it can't allocate 2MB large pages (if Memory::MF_HUGE_HINT is provided) Currently, Memory::MF_HUGE_HINT only works on Windows. The hint will be ignored on Linux, 4KB pages will always be returned. Differential Revision: https://reviews.llvm.org/D58718 llvm-svn: 355065
1 parent d4b4e17 commit b05ba93

File tree

4 files changed

+77
-21
lines changed

4 files changed

+77
-21
lines changed

llvm/include/llvm/Support/Memory.h

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ namespace sys {
3535
private:
3636
void *Address; ///< Address of first byte of memory area
3737
size_t Size; ///< Size, in bytes of the memory area
38+
unsigned Flags = 0;
3839
friend class Memory;
3940
};
4041

@@ -45,9 +46,11 @@ namespace sys {
4546
class Memory {
4647
public:
4748
enum ProtectionFlags {
48-
MF_READ = 0x1000000,
49+
MF_READ = 0x1000000,
4950
MF_WRITE = 0x2000000,
50-
MF_EXEC = 0x4000000
51+
MF_EXEC = 0x4000000,
52+
MF_RWE_MASK = 0x7000000,
53+
MF_HUGE_HINT = 0x0000001
5154
};
5255

5356
/// This method allocates a block of memory that is suitable for loading

llvm/lib/Support/Unix/Memory.inc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ extern "C" void __clear_cache(void *, void*);
4545
namespace {
4646

4747
int getPosixProtectionFlags(unsigned Flags) {
48-
switch (Flags) {
48+
switch (Flags & llvm::sys::Memory::MF_RWE_MASK) {
4949
case llvm::sys::Memory::MF_READ:
5050
return PROT_READ;
5151
case llvm::sys::Memory::MF_WRITE:
@@ -114,6 +114,7 @@ Memory::allocateMappedMemory(size_t NumBytes,
114114
if (Start && Start % PageSize)
115115
Start += PageSize - Start % PageSize;
116116

117+
// FIXME: Handle huge page requests (MF_HUGE_HINT).
117118
void *Addr = ::mmap(reinterpret_cast<void *>(Start), NumBytes, Protect,
118119
MMFlags, fd, 0);
119120
if (Addr == MAP_FAILED) {

llvm/lib/Support/Windows/Memory.inc

Lines changed: 54 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
namespace {
2323

2424
DWORD getWindowsProtectionFlags(unsigned Flags) {
25-
switch (Flags) {
25+
switch (Flags & llvm::sys::Memory::MF_RWE_MASK) {
2626
// Contrary to what you might expect, the Windows page protection flags
2727
// are not a bitwise combination of RWX values
2828
case llvm::sys::Memory::MF_READ:
@@ -47,6 +47,9 @@ DWORD getWindowsProtectionFlags(unsigned Flags) {
4747
return PAGE_NOACCESS;
4848
}
4949

50+
// While we'd be happy to allocate single pages, the Windows allocation
51+
// granularity may be larger than a single page (in practice, it is 64K)
52+
// so mapping less than that will create an unreachable fragment of memory.
5053
size_t getAllocationGranularity() {
5154
SYSTEM_INFO Info;
5255
::GetSystemInfo(&Info);
@@ -56,6 +59,38 @@ size_t getAllocationGranularity() {
5659
return Info.dwAllocationGranularity;
5760
}
5861

62+
// Large/huge memory pages need explicit process permissions in order to be
63+
// used. See https://blogs.msdn.microsoft.com/oldnewthing/20110128-00/?p=11643
64+
// Also large pages need to be manually enabled on your OS. If all this is
65+
// sucessfull, we return the minimal large memory page size.
66+
static size_t enableProcessLargePages() {
67+
HANDLE Token = 0;
68+
size_t LargePageMin = GetLargePageMinimum();
69+
if (LargePageMin)
70+
OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY,
71+
&Token);
72+
if (!Token)
73+
return 0;
74+
LUID Luid;
75+
if (!LookupPrivilegeValue(0, SE_LOCK_MEMORY_NAME, &Luid)) {
76+
CloseHandle(Token);
77+
return 0;
78+
}
79+
TOKEN_PRIVILEGES TP{};
80+
TP.PrivilegeCount = 1;
81+
TP.Privileges[0].Luid = Luid;
82+
TP.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
83+
if (!AdjustTokenPrivileges(Token, FALSE, &TP, 0, 0, 0)) {
84+
CloseHandle(Token);
85+
return 0;
86+
}
87+
DWORD E = GetLastError();
88+
CloseHandle(Token);
89+
if (E == ERROR_SUCCESS)
90+
return LargePageMin;
91+
return 0;
92+
}
93+
5994
} // namespace
6095

6196
namespace llvm {
@@ -74,19 +109,20 @@ MemoryBlock Memory::allocateMappedMemory(size_t NumBytes,
74109
if (NumBytes == 0)
75110
return MemoryBlock();
76111

77-
// While we'd be happy to allocate single pages, the Windows allocation
78-
// granularity may be larger than a single page (in practice, it is 64K)
79-
// so mapping less than that will create an unreachable fragment of memory.
80-
// Avoid using one-time initialization of static locals here, since they
81-
// aren't thread safe with MSVC.
82-
static volatile size_t GranularityCached;
83-
size_t Granularity = GranularityCached;
84-
if (Granularity == 0) {
85-
Granularity = getAllocationGranularity();
86-
GranularityCached = Granularity;
112+
static size_t DefaultGranularity = getAllocationGranularity();
113+
static Optional<size_t> LargePageGranularity = enableProcessLargePages();
114+
115+
DWORD AllocType = MEM_RESERVE | MEM_COMMIT;
116+
bool HugePages = false;
117+
size_t Granularity = DefaultGranularity;
118+
119+
if ((Flags & MF_HUGE_HINT) && LargePageGranularity.hasValue()) {
120+
AllocType |= MEM_LARGE_PAGES;
121+
HugePages = true;
122+
Granularity = *LargePageGranularity;
87123
}
88124

89-
const size_t NumBlocks = (NumBytes+Granularity-1)/Granularity;
125+
size_t NumBlocks = (NumBytes + Granularity - 1) / Granularity;
90126

91127
uintptr_t Start = NearBlock ? reinterpret_cast<uintptr_t>(NearBlock->base()) +
92128
NearBlock->size()
@@ -99,13 +135,12 @@ MemoryBlock Memory::allocateMappedMemory(size_t NumBytes,
99135

100136
DWORD Protect = getWindowsProtectionFlags(Flags);
101137

102-
void *PA = ::VirtualAlloc(reinterpret_cast<void*>(Start),
103-
NumBlocks*Granularity,
104-
MEM_RESERVE | MEM_COMMIT, Protect);
138+
void *PA = ::VirtualAlloc(reinterpret_cast<void *>(Start),
139+
NumBlocks * Granularity, AllocType, Protect);
105140
if (PA == NULL) {
106-
if (NearBlock) {
107-
// Try again without the NearBlock hint
108-
return allocateMappedMemory(NumBytes, NULL, Flags, EC);
141+
if (NearBlock || HugePages) {
142+
// Try again without the NearBlock hint and without large memory pages
143+
return allocateMappedMemory(NumBytes, NULL, Flags & ~MF_HUGE_HINT, EC);
109144
}
110145
EC = mapWindowsError(::GetLastError());
111146
return MemoryBlock();
@@ -114,6 +149,7 @@ MemoryBlock Memory::allocateMappedMemory(size_t NumBytes,
114149
MemoryBlock Result;
115150
Result.Address = PA;
116151
Result.Size = NumBlocks*Granularity;
152+
Result.Flags = (Flags & ~MF_HUGE_HINT) | (HugePages ? MF_HUGE_HINT : 0);
117153

118154
if (Flags & MF_EXEC)
119155
Memory::InvalidateInstructionCache(Result.Address, Result.Size);

llvm/unittests/Support/MemoryTest.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,22 @@ TEST_P(MappedMemoryTest, AllocAndRelease) {
105105
EXPECT_FALSE(Memory::releaseMappedMemory(M1));
106106
}
107107

108+
TEST_P(MappedMemoryTest, AllocAndReleaseHuge) {
109+
CHECK_UNSUPPORTED();
110+
std::error_code EC;
111+
MemoryBlock M1 = Memory::allocateMappedMemory(
112+
sizeof(int), nullptr, Flags | Memory::MF_HUGE_HINT, EC);
113+
EXPECT_EQ(std::error_code(), EC);
114+
115+
// Test large/huge memory pages. In the worst case, 4kb pages should be
116+
// returned, if large pages aren't available.
117+
118+
EXPECT_NE((void *)nullptr, M1.base());
119+
EXPECT_LE(sizeof(int), M1.size());
120+
121+
EXPECT_FALSE(Memory::releaseMappedMemory(M1));
122+
}
123+
108124
TEST_P(MappedMemoryTest, MultipleAllocAndRelease) {
109125
CHECK_UNSUPPORTED();
110126
std::error_code EC;

0 commit comments

Comments
 (0)