-
Notifications
You must be signed in to change notification settings - Fork 0
/
datastream.lua
516 lines (451 loc) · 13.2 KB
/
datastream.lua
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
--[=[
-- Medium-sized streams library by Vurv78
-- Credit to RPFeltz (https://stackoverflow.com/questions/14416734/lua-packing-ieee754-single-precision-floating-point-numbers) for the floating point number implementation.
-- Rest is by me.
-- Fully documented with EmmyLua annotations.
## Features
Reading & Writing:
Null terminated strings
Signed Integers (8, 16, 32, 64)
Unsigned Integers (8, 16, 32, 64)
Set length strings
Custom Structure system:
There are custom structures you can create using the DataStruct type.
This type abstracts the DataStream type by allowing you to form structs in a nice,
visual pattern similar to C structs or whever else you'd normally use structs.
See an example below.
## Todo
IEEE754 Binary64 Double (F64)
IEEE754 Binary16 Float (F16)
Posit Numbers / Universal Numbers (http://www.johngustafson.net/pdfs/BeatingFloatingPoint.pdf)
]=]
local I8_MAX = 128
local I16_MAX = 32768
local I32_MAX = 2147483648
local I64_MAX = 9223372036854775808
local U8_MAX = 256
local U16_MAX = 65536
local U32_MAX = 4294967296
local U64_MAX = 18446744073709551616
local MAX = {
[8] = I8_MAX,
[16] = I16_MAX,
[32] = I32_MAX,
[64] = I64_MAX,
}
local UMAX = {
[8] = U8_MAX,
[16] = U16_MAX,
[32] = U32_MAX,
[64] = U64_MAX,
}
--- Internal class used by the DataStruct.
--- Much better for lightweight data structures, or for missing features of the Datastruct (e.g. set length strings)
---@class DataStream
---@field content string # Reader only
---@field ptr integer # Reader only
---@field len number # Reader only
---@field index number # Writer only
---@field parts string # Writer only
local DataStream = {}
DataStream.__index = DataStream
---@param str string
---@return DataStream
function DataStream.new(str)
return setmetatable({
content = str,
ptr = 0,
len = str and #str or 0,
parts = {},
index = 0
}, DataStream)
end
function DataStream:__tostring()
return "DataStream"
end
---@return number u8
function DataStream:readU8()
self.ptr = self.ptr + 1
return string.byte(self.content, self.ptr)
end
--- Reads an unsigned integer from the stream
---@param n integer # 8, 16, 32, 64 ...
---@return number
function DataStream:readU(n)
local bytes = n / 8
local out = 0
for i = 0, bytes - 1 do
local b = self:readU8()
out = out + bit.lshift(b, 8 * i)
end
return out
end
--- Reads a signed 8 bit integer (-128, 128) from the stream
---@return number Int8 at this position
function DataStream:readI8()
local x = self:readU8()
if x >= I8_MAX then x = x - U8_MAX end
return x
end
--- Reads a signed 8 bit integer (-128, 128) from the stream
---@return number Int8 at this position
function DataStream:readI16()
local x = self:readU(16)
if x >= I16_MAX then x = x - U16_MAX end
return x
end
--- Reads a signed 8 bit integer (-128, 128) from the stream
---@param n integer # 8, 16, 32, 64 ...
---@return number Int8 at this position
function DataStream:readI(n)
local val = self:readU(n)
if val >= MAX[n] then val = val - UMAX[n] end
return val
end
---@return string
function DataStream:readString()
return self:readUntil(0)
end
--- Reads a IEEE754 Float32 from the stream and returns it
---@return number float
function DataStream:readF32()
local b1, b2, b3, b4 = string.byte(self.content, self.ptr + 1, self.ptr + 4)
self.ptr = self.ptr + 4
local exponent = (b1 % 0x80) * 0x02 + math.floor(b2 / 0x80)
local mantissa = math.ldexp(((b2 % 0x80) * 0x100 + b3) * 0x100 + b4, -23)
if exponent == 0xFF then
if mantissa > 0 then
return 0 / 0
else
mantissa = math.huge
exponent = 0x7F
end
elseif exponent > 0 then
mantissa = mantissa + 1
else
exponent = exponent + 1
end
if b1 >= 0x80 then
mantissa = -mantissa
end
return math.ldexp(mantissa, exponent - 0x7F)
end
---@param len integer
function DataStream:read(len)
local start = self.ptr + 1
self.ptr = self.ptr + len
return string.sub(self.content, start, self.ptr)
end
---@param byte integer
---@return string
function DataStream:readUntil(byte)
self.ptr = self.ptr + 1
local ed = string.find(self.content, string.char(byte), self.ptr, true) or #self.content
local ret = string.sub(self.content, self.ptr, ed - 1)
self.ptr = ed
return ret
end
---@param str string
---@param not_terminated boolean # Whether to append a null char to the end, to make this able to be read with readString. (Else, will need to be self:read())
function DataStream:writeString(str, not_terminated)
self.index = self.index + 1
self.parts[self.index] = str
if not not_terminated then
self:writeU8(0)
end
end
--- Writes vararg bytes to the stream, connecting them with table.concat and string.char
function DataStream:write(...)
self.index = self.index + 1
self.parts[self.index] = table.concat {string.char(...)}
end
-- Packs a IEEE754 Float32 into 4 U8s and writes it into the buffer
---@param float number
function DataStream:writeF32(float)
if float == 0 then
self:write(0, 0, 0, 0)
elseif float ~= float then
self:write(0xFF, 0xFF, 0xFF, 0xFF)
else
local sign = 0x00
if float < 0 then
sign = 0x80
float = -float
end
local mantissa, exponent = math.frexp(float)
exponent = exponent + 0x7F
if exponent <= 0 then
mantissa = math.ldexp(mantissa, exponent - 1)
exponent = 0
elseif exponent > 0 then
if exponent >= 0xFF then
self:write(sign + 0x7F, 0x80, 0x00, 0x00)
elseif exponent == 1 then
exponent = 0
else
mantissa = mantissa * 2 - 1
exponent = exponent - 1
end
end
mantissa = math.floor(math.ldexp(mantissa, 23) + 0.5)
self:write(
sign + math.floor(exponent / 2),
(exponent % 2) * 0x80 + math.floor(mantissa / 0x10000),
math.floor(mantissa / 0x100) % 0x100,
mantissa % 0x100
)
end
end
---@param byte integer
function DataStream:writeU8(byte)
self.index = self.index + 1
self.parts[self.index] = string.char(byte)
end
---@param n integer
function DataStream:writeU16(n)
self:writeU8( n % U8_MAX )
self:writeU8( math.floor(n / U8_MAX) )
end
---@param n integer
function DataStream:writeU32(n)
self:writeU16( n % U16_MAX )
self:writeU16( math.floor(n / U16_MAX) )
end
---@param n integer
function DataStream:writeU64(n)
self:writeU32( n % U32_MAX )
self:writeU32( math.floor(n / U32_MAX) )
end
--- Writes a signed byte to the buffer.
--- Note this will not handle cases where the number is infinity or nan.
---@param n integer
function DataStream:writeI8(n)
if n < 0 then n = n + U8_MAX end
self:writeU8(n % U8_MAX)
end
--- Writes a signed 16 bit integer to the buffer.
--- Note this will not handle cases where the number is infinity or nan.
---@param n integer
function DataStream:writeI16(n)
if n < 0 then n = n + U16_MAX end
self:write(
n % U8_MAX,
bit.rshift(n, 8) % U8_MAX
)
end
--- Writes a signed 32 bit integer to the buffer.
--- Note this will not handle cases where the number is infinity or nan.
---@param n integer
function DataStream:writeI32(n)
if n < 0 then n = n + U32_MAX end
self:write(
n % U8_MAX,
bit.rshift(n, 8) % U8_MAX,
bit.rshift(n, 16) % U8_MAX,
bit.rshift(n, 24) % U8_MAX
)
end
--- Writes a signed 64 bit integer to the buffer.
--- Note this will not handle cases where the number is infinity or nan.
---@param n integer
function DataStream:writeI64(n)
if n < 0 then n = n + U64_MAX end
self:write(
n % U8_MAX,
bit.rshift(n, 8) % U8_MAX,
bit.rshift(n, 16) % U8_MAX,
bit.rshift(n, 24) % U8_MAX,
bit.rshift(n, 32) % U8_MAX,
bit.rshift(n, 40) % U8_MAX,
bit.rshift(n, 48) % U8_MAX,
bit.rshift(n, 56) % U8_MAX
)
end
---@return string
function DataStream:getBuffer()
return table.concat(self.parts)
end
--- End of DataStream class, now for the DataStruct helper.
--- Helper Struct class to Read and Write structs from a struct definition
--- ## Example code
--- ```lua
--- ---@type DataStruct
--- local Message = DataStruct [[
--- // Comments (C, Lua or Python style.)
--- sender_id: u32;
--- message: cstr;
--- // Variable length array / Explicit vector.
--- n_reactions: u32;
--- reactions: [u8; $n_reactions]
--- ]]
--- local msg = Message:encode {
--- sender_id = 55,
--- message = "Foobar",
--- n_reactions = 3,
--- reactions = { 2, 5, 23 }
--- }
---
--- local bytes = msg:getBuffer()
---
--- local msg = Message:decode( bytes )
--- for k, v in pairs(msg) do
--- print(k, v)
--- end
--- ```
---@class DataStruct
---@field fields table<number, { key: string, rtype: string, count: string }>
---@field data table<number, any>
---@field n integer # Number of fields in the struct
---@field stream DataStream # Internal data stream used for encoding types to bytes.
local DataStruct = {}
DataStruct.__index = DataStruct
function DataStruct:__tostring()
return "DataStruct #" .. self.n
end
---@type table<string, fun(self: DataStruct): number|string>
local Handlers = {
["i8"] = function(self) return self:readI(8) end,
["i16"] = function(self) return self:readI(16) end,
["i32"] = function(self) return self:readI(32) end,
["u8"] = function(self) return self:readU(8) end,
["u16"] = function(self) return self:readU(16) end,
["u32"] = function(self) return self:readU(32) end,
["cstr"] = function(self) return self:readString() end,
["f32"] = function(self) return self:readF32() end,
["f64"] = function(self) error("Not implemented") end
}
local WriteHandlers = {
["i8"] = function(self, value) self:writeI8(value) end,
["i16"] = function(self, value) self:writeI16(value) end,
["i32"] = function(self, value) self:writeI32(value) end,
["u8"] = function(self, value) self:writeU8(value) end,
["u16"] = function(self, value) self:writeU16(value) end,
["u32"] = function(self, value) self:writeU32(value) end,
["cstr"] = function(self, value) self:writeString(value) end,
["f32"] = function(self, value) self:writeF32(value) end,
["f64"] = function(self, value) error("Not implemented") end
}
---@param definition string
local function parse(definition)
local nocomments = string.gsub(definition, "[#/-]+.-\n", "\n")
local struct, n = {}, 1
for line in nocomments:gmatch("[^\n\r,]+") do
-- Parser? Nah :p
local key, rtype, count = line:match("%s*([%w_]+)%s*[:=]%s*%[?%s*([uifcstr]+%d*);?%s*%$?([%w_]*)%]?")
-- Check if key exists, because an empty line being passed here would break it otherwise.
-- Comments cause this.
if key then
if struct[key] then
error("Repeated key [" .. key .. "] found at line " .. n .. " in DataStruct builder")
end
local handler = Handlers[rtype]
if not handler then error("Unknown or invalid type [".. rtype .. "] in DataStruct builder") end
-- "count" is the number of times to read the type.
-- This is for array types [f32; 3].
-- It may be a string, in case of variable length. [f32; $n]
if count == "" then
count = nil
assert( not string.find(line, "[", 1, true), "Malformed array block. Use [u8; 55] or [i32; $len]" )
else
local n = tonumber(count)
if not n then
assert(struct[count], "Array length field $" .. count .. " not found in DataStruct builder")
else
count = n
end
end
local t = { key, n, rtype, count }
struct[key], struct[n] = t, t
n = n + 1
end
end
return struct, n
end
---@param str string
function DataStruct.new(str)
local def, n = parse(str)
return setmetatable({
fields = def,
n = n,
stream = DataStream.new(),
-- Read data
data = {},
}, DataStruct)
end
--- Encodes data into bytes, from a DataStruct template.
---@param data table<string|number, any>
---@return DataStream writer # Writing stream, use :getBuffer() to get the bytes.
function DataStruct:encode(data)
local out, writer = {}, DataStream.new()
for k, v in pairs(self.fields) do
local idx = v[1]
assert(data[idx], "Missing field [" .. idx .. "] in DataStruct:encode")
end
for k, v in pairs(data) do
local field = self.fields[k]
if field then
if type(k) == "number" then
out[k] = v
else
local idx = field[2]
out[idx] = v
end
end
end
for k, v in ipairs(out) do
local field = self.fields[k]
local ty, count = field[3], field[4]
if type(count) == "number" then
for i = 1, count do
WriteHandlers[ty](writer, v[i])
end
elseif count then
-- Variable length
local count = assert(data[count], "Missing variable reference [" .. count .. "] in DataStruct:encode")
for i = 1, count do
WriteHandlers[ty](writer, v[i])
end
else
-- Single item, no table
WriteHandlers[ty](writer, v)
end
end
return writer
end
--- Decodes data given a string of bytes.
---@param stream string
---@return table<string, any>
function DataStruct:decode(stream)
local reader = DataStream.new(stream)
local out = {}
for nidx, v in ipairs(self.fields) do
local idx, ty, count = v[1], v[3], v[4]
if type(count) == "number" then
local t = {}
for i = 1, count do
t[i] = Handlers[ty](reader)
end
out[idx] = t
elseif count then
-- Variable length
local v = self.fields[count][1]
assert(out[v], "Variable length field [" .. count .. "] not found at runtime")
local count = out[v]
assert(type(count) == "number", "Variable length field [" .. count .. "] is not a number")
local t = {}
for i = 1, count do
t[i] = Handlers[ty](reader)
end
out[idx] = t
else
-- Single item
out[idx] = Handlers[ty](reader)
end
end
return out
end
---@return string?
function DataStruct:getBuffer()
return self.stream:getBuffer()
end
return DataStream, DataStruct.new, DataStruct