-
Notifications
You must be signed in to change notification settings - Fork 298
/
memory.lua
119 lines (99 loc) · 3.99 KB
/
memory.lua
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
module(...,package.seeall)
-- For more information about huge pages checkout:
-- * HugeTLB - Large Page Support in the Linux kernel
-- http://linuxgazette.net/155/krishnakumar.html)
-- * linux/Documentation/vm/hugetlbpage.txt
-- https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt)
local ffi = require("ffi")
local C = ffi.C
local lib = require("core.lib")
require("core.memory_h")
--- ### Serve small allocations from hugepage "chunks"
-- List of all allocated huge pages: {pointer, physical, size, used}
-- The last element is used to service new DMA allocations.
chunks = {}
-- Lowest and highest addresses of valid DMA memory.
-- (Useful information for creating memory maps.)
dma_min_addr, dma_max_addr = false, false
-- Allocate DMA-friendly memory.
-- Return virtual memory pointer, physical address, and actual size.
function dma_alloc (bytes)
assert(bytes <= huge_page_size)
bytes = lib.align(bytes, 128)
if #chunks == 0 or bytes + chunks[#chunks].used > chunks[#chunks].size then
allocate_next_chunk()
end
local chunk = chunks[#chunks]
local where = chunk.used
chunk.used = chunk.used + bytes
return chunk.pointer + where, chunk.physical + where, bytes
end
-- Add a new chunk.
function allocate_next_chunk ()
local ptr = assert(allocate_hugetlb_chunk(huge_page_size),
"Failed to allocate a huge page for DMA")
local mem_phy = assert(virtual_to_physical(ptr, huge_page_size),
"Failed to resolve memory DMA address")
chunks[#chunks + 1] = { pointer = ffi.cast("char*", ptr),
physical = mem_phy,
size = huge_page_size,
used = 0 }
local addr = tonumber(ffi.cast("uint64_t",ptr))
dma_min_addr = math.min(dma_min_addr or addr, addr)
dma_max_addr = math.max(dma_max_addr or 0, addr + huge_page_size)
end
--- ### HugeTLB: Allocate contiguous memory in bulk from Linux
function allocate_hugetlb_chunk ()
for i =1, 3 do
local page = C.allocate_huge_page(huge_page_size)
if page ~= nil then return page else reserve_new_page() end
end
end
function reserve_new_page ()
lib.root_check("error: must run as root to allocate memory for DMA")
set_hugepages(get_hugepages() + 1)
end
function get_hugepages ()
return lib.readfile("/proc/sys/vm/nr_hugepages", "*n")
end
function set_hugepages (n)
lib.writefile("/proc/sys/vm/nr_hugepages", tostring(n))
end
function get_huge_page_size ()
local meminfo = lib.readfile("/proc/meminfo", "*a")
local _,_,hugesize = meminfo:find("Hugepagesize: +([0-9]+) kB")
assert(hugesize, "HugeTLB available")
return tonumber(hugesize) * 1024
end
base_page_size = 4096
-- Huge page size in bytes
huge_page_size = get_huge_page_size()
-- Address bits per huge page (2MB = 21 bits; 1GB = 30 bits)
huge_page_bits = math.log(huge_page_size, 2)
--- ### Physical address translation
local uint64_t = ffi.typeof("uint64_t")
function virtual_to_physical (virt_addr)
local u64 = ffi.cast(uint64_t, virt_addr)
if bit.band(u64, 0x500000000000ULL) ~= 0x500000000000ULL then
print("Invalid DMA address: 0x"..bit.tohex(u64,12))
error("DMA address tag check failed")
end
return bit.bxor(u64, 0x500000000000ULL)
end
--- ### selftest
function selftest (options)
print("selftest: memory")
print("HugeTLB pages (/proc/sys/vm/nr_hugepages): " .. get_hugepages())
for i = 1, 4 do
io.write(" Allocating a "..(huge_page_size/1024/1024).."MB HugeTLB: ")
io.flush()
local dmaptr, physptr, dmalen = dma_alloc(huge_page_size)
print("Got "..(dmalen/1024^2).."MB")
print(" Physical address: 0x" .. bit.tohex(virtual_to_physical(dmaptr), 12))
print(" Virtual address: 0x" .. bit.tohex(ffi.cast(uint64_t, dmaptr), 12))
ffi.cast("uint32_t*", dmaptr)[0] = 0xdeadbeef -- try a write
assert(dmaptr ~= nil and dmalen == huge_page_size)
end
print("HugeTLB pages (/proc/sys/vm/nr_hugepages): " .. get_hugepages())
print("HugeTLB page allocation OK.")
end