-
Notifications
You must be signed in to change notification settings - Fork 298
/
packet.lua
369 lines (321 loc) · 11.2 KB
/
packet.lua
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
-- Use of this source code is governed by the Apache 2.0 license; see COPYING.
module(...,package.seeall)
local debug = _G.developer_debug
local ffi = require("ffi")
local bit = require("bit")
local C = ffi.C
local lib = require("core.lib")
local memory = require("core.memory")
local shm = require("core.shm")
local counter = require("core.counter")
local sync = require("core.sync")
local timeline = require("core.timeline")
require("core.packet_h")
local packet_t = ffi.typeof("struct packet")
local packet_ptr_t = ffi.typeof("struct packet *")
local packet_size = ffi.sizeof(packet_t)
max_payload = tonumber(C.PACKET_PAYLOAD_SIZE)
-- For operations that add or remove headers from the beginning of a
-- packet, instead of copying around the payload we just move the
-- packet structure as a whole around.
local packet_alignment = 512
local default_headroom = 256
-- The Intel82599 driver requires even-byte alignment, so let's keep
-- things aligned at least this much.
local minimum_alignment = 2
local function get_alignment (addr, alignment)
-- Precondition: alignment is a power of 2.
return bit.band(addr, alignment - 1)
end
local function get_headroom (ptr)
return get_alignment(ffi.cast("uint64_t", ptr), packet_alignment)
end
local function is_aligned (addr, alignment)
return get_alignment(addr, alignment) == 0
end
local function headroom_valid (headroom)
return 0 <= headroom and headroom < packet_alignment
and is_aligned(headroom, minimum_alignment)
end
-- Freelist containing empty packets ready for use.
local max_packets = 1e6
ffi.cdef([[
struct freelist {
int32_t lock[1];
uint64_t nfree;
uint64_t max;
struct packet *list[]]..max_packets..[[];
};
]])
local function freelist_create(name)
local fl = shm.create(name, "struct freelist")
fl.max = max_packets
return fl
end
local function freelist_open(name, readonly)
return shm.open(name, "struct freelist", readonly)
end
local function freelist_full(freelist)
return freelist.nfree == freelist.max
end
local function freelist_add(freelist, element)
-- Safety check
if _G.developer_debug then
assert(not freelist_full(freelist), "freelist overflow")
end
freelist.list[freelist.nfree] = element
freelist.nfree = freelist.nfree + 1
end
local function freelist_remove(freelist)
if freelist.nfree == 0 then
error("no free packets")
else
freelist.nfree = freelist.nfree - 1
return freelist.list[freelist.nfree]
end
end
local function freelist_nfree(freelist)
return freelist.nfree
end
local function freelist_lock(freelist)
sync.lock(freelist.lock)
end
local function freelist_unlock(freelist)
sync.unlock(freelist.lock)
end
local packet_allocation_step = 1000
local packets_allocated = 0
-- Initialized on demand.
local packets_fl, group_fl, events
-- Call to ensure packet freelist is enabled.
function initialize ()
packets_fl = freelist_create("engine/packets.freelist")
events = timeline.load_events(engine.timeline(), "core.packet")
end
-- Call to ensure group freelist is enabled.
function enable_group_freelist ()
if not group_fl then
group_fl = freelist_create("group/packets.freelist")
end
end
-- Return borrowed packets to group freelist.
function rebalance_freelists ()
if group_fl and freelist_nfree(packets_fl) > packets_allocated then
events.group_freelist_wait()
freelist_lock(group_fl)
events.group_freelist_locked()
local nfree0 = freelist_nfree(packets_fl)
while freelist_nfree(packets_fl) > packets_allocated
and not freelist_full(group_fl) do
freelist_add(group_fl, freelist_remove(packets_fl))
end
events.group_freelist_released(nfree0 - freelist_nfree(packets_fl))
freelist_unlock(group_fl)
events.group_freelist_unlocked()
end
end
-- Return an empty packet.
function allocate ()
if freelist_nfree(packets_fl) == 0 then
if group_fl then
events.group_freelist_wait()
freelist_lock(group_fl)
events.group_freelist_locked()
while freelist_nfree(group_fl) > 0
and freelist_nfree(packets_fl) < packets_allocated do
freelist_add(packets_fl, freelist_remove(group_fl))
end
freelist_unlock(group_fl)
events.group_freelist_unlocked()
events.group_freelist_reclaimed(freelist_nfree(packets_fl))
end
if freelist_nfree(packets_fl) == 0 then
preallocate_step()
end
end
events.packet_allocated()
return freelist_remove(packets_fl)
end
-- Release all packets allocated by pid to its group freelist (if one exists.)
--
-- This is an internal API function provided for cleanup during
-- process termination.
function shutdown (pid)
local in_group, group_fl = pcall(
freelist_open, "/"..pid.."/group/packets.freelist"
)
if in_group then
local packets_fl = freelist_open("/"..pid.."/engine/packets.freelist")
freelist_lock(group_fl)
while freelist_nfree(packets_fl) > 0 do
freelist_add(group_fl, freelist_remove(packets_fl))
end
freelist_unlock(group_fl)
end
end
-- Create a new empty packet.
function new_packet ()
local base = memory.dma_alloc(packet_size + packet_alignment,
packet_alignment)
local p = ffi.cast(packet_ptr_t, base + default_headroom)
p.length = 0
return p
end
-- Create an exact copy of a packet.
function clone (p)
return from_pointer(p.data, p.length)
end
-- Append data to the end of a packet.
function append (p, ptr, len)
assert(p.length + len <= max_payload, "packet payload overflow")
ffi.copy(p.data + p.length, ptr, len)
p.length = p.length + len
return p
end
-- Prepend data to the start of a packet.
function prepend (p, ptr, len)
p = shiftright(p, len)
ffi.copy(p.data, ptr, len) -- Fill the gap
return p
end
-- Move packet data to the left. This shortens the packet by dropping
-- the header bytes at the front.
function shiftleft (p, bytes)
assert(0 <= bytes and bytes <= p.length)
local ptr = ffi.cast("char*", p)
local len = p.length
local headroom = get_headroom(ptr)
if headroom_valid(bytes + headroom) then
-- Fast path: just shift the packet pointer.
p = ffi.cast(packet_ptr_t, ptr + bytes)
p.length = len - bytes
return p
else
-- Slow path: shift packet data, resetting the default headroom.
local delta_headroom = default_headroom - headroom
C.memmove(p.data + delta_headroom, p.data + bytes, len - bytes)
p = ffi.cast(packet_ptr_t, ptr + delta_headroom)
p.length = len - bytes
return p
end
end
-- Move packet data to the right. This leaves length bytes of data
-- at the beginning of the packet.
function shiftright (p, bytes)
local ptr = ffi.cast("char*", p)
local len = p.length
local headroom = get_headroom(ptr)
if headroom_valid(headroom - bytes) then
-- Fast path: just shift the packet pointer.
p = ffi.cast(packet_ptr_t, ptr - bytes)
p.length = len + bytes
return p
else
-- Slow path: shift packet data, resetting the default headroom.
assert(bytes <= max_payload - len)
local delta_headroom = default_headroom - headroom
C.memmove(p.data + bytes + delta_headroom, p.data, len)
p = ffi.cast(packet_ptr_t, ptr + delta_headroom)
p.length = len + bytes
return p
end
end
-- Conveniently create a packet by copying some existing data.
function from_pointer (ptr, len) return append(allocate(), ptr, len) end
function from_string (d) return from_pointer(d, #d) end
-- Free a packet that is no longer in use.
local function free_internal (p)
local ptr = ffi.cast("char*", p)
p = ffi.cast(packet_ptr_t, ptr - get_headroom(ptr) + default_headroom)
p.length = 0
freelist_add(packets_fl, p)
end
function account_free (p)
counter.add(engine.frees)
counter.add(engine.freebytes, p.length)
-- Calculate bits of physical capacity required for packet on 10GbE
-- Account for minimum data size and overhead of CRC and inter-packet gap
counter.add(engine.freebits, (math.max(p.length, 46) + 4 + 5) * 8)
end
function free (p)
events.packet_freed(p.length)
account_free(p)
free_internal(p)
end
-- Set packet data length.
function resize (p, len)
assert(len <= max_payload, "packet payload overflow")
ffi.fill(p.data + p.length, math.max(0, len - p.length))
p.length = len
return p
end
function preallocate_step()
assert(packets_allocated + packet_allocation_step <= max_packets,
"packet allocation overflow")
for i=1, packet_allocation_step do
free_internal(new_packet(), true)
end
packets_allocated = packets_allocated + packet_allocation_step
packet_allocation_step = 2 * packet_allocation_step
events.packets_preallocated(packet_allocation_step)
end
function selftest ()
assert(is_aligned(0, 1))
assert(is_aligned(1, 1))
assert(is_aligned(2, 1))
assert(is_aligned(3, 1))
assert( is_aligned(0, 2))
assert(not is_aligned(1, 2))
assert( is_aligned(2, 2))
assert(not is_aligned(3, 2))
assert( is_aligned(0, 512))
assert(not is_aligned(1, 512))
assert(not is_aligned(2, 512))
assert(not is_aligned(3, 512))
assert(not is_aligned(510, 512))
assert(not is_aligned(511, 512))
assert( is_aligned(512, 512))
assert(not is_aligned(513, 512))
local function is_power_of_2 (x) return bit.band(x, x-1) == 0 end
assert(is_power_of_2(minimum_alignment))
assert(is_power_of_2(packet_alignment))
assert(is_aligned(default_headroom, minimum_alignment))
local function check_free (p)
free(p)
-- Check that the last packet added to the free list has the
-- default headroom.
local p = allocate()
assert(get_headroom(p) == default_headroom)
free(p)
end
local function check_shift(init_len, shift, amount, len, headroom)
local p = allocate()
p.length = init_len
p = shift(p, amount)
assert(p.length == len)
assert(get_headroom(p) == headroom)
check_free(p)
end
local function check_fast_shift(init_len, shift, amount, len, headroom)
assert(headroom_valid(amount))
check_shift(init_len, shift, amount, len, headroom)
end
local function check_slow_shift(init_len, shift, amount, len)
check_shift(init_len, shift, amount, len, default_headroom)
end
check_fast_shift(0, function (p, amt) return p end, 0, 0, default_headroom)
check_fast_shift(0, shiftright, 0, 0, default_headroom)
check_fast_shift(0, shiftright, 10, 10, default_headroom - 10)
check_slow_shift(0, shiftright, 11, 11)
check_fast_shift(512, shiftleft, 0, 512, default_headroom)
check_fast_shift(512, shiftleft, 10, 502, default_headroom + 10)
check_slow_shift(512, shiftleft, 11, 501)
check_fast_shift(0, shiftright, default_headroom, default_headroom, 0)
check_slow_shift(0, shiftright, default_headroom + 2, default_headroom + 2)
check_slow_shift(0, shiftright, packet_alignment * 2, packet_alignment * 2)
check_fast_shift(packet_alignment, shiftleft,
packet_alignment - default_headroom - 2,
default_headroom + 2, packet_alignment - 2)
check_slow_shift(packet_alignment, shiftleft,
packet_alignment - default_headroom, default_headroom)
end