Skip to content
This repository has been archived by the owner on Aug 25, 2020. It is now read-only.

Stream interface #3

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Open

Stream interface #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
6b2e369
Async-ify all encoding
fengb Aug 23, 2019
ba7d45a
Restructured streaming_encode
fengb Aug 24, 2019
91085ba
Localize suspend point
fengb Aug 24, 2019
0d0364b
Convert BytesCoder into using Writer.write()
fengb Aug 24, 2019
eaee229
Switch from AsyncContext into OutStream interface
fengb Aug 25, 2019
bf9205e
Pushed IO into separate file
fengb Aug 26, 2019
12938dd
Use .value over .data
fengb Aug 26, 2019
568d222
Use std.io builtins
fengb Aug 26, 2019
a2af096
Switch decodeFrom to use streaming
fengb Aug 26, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
163 changes: 72 additions & 91 deletions src/coder.zig
View file
Open in desktop
@@ -1,12 +1,6 @@
const std = @import("std");
const testing = std.testing;

pub const ParseError = error{
Overflow,
EndOfStream,
OutOfMemory,
};

pub const Uint64Coder = struct {
pub const primitive = u64;

Expand All @@ -15,49 +9,41 @@ pub const Uint64Coder = struct {
return std.math.max(divCeil(u64, bits, 7), 1);
}

pub fn encode(buffer: []u8, data: u64) []u8 {
if (data == 0) {
buffer[0] = 0;
return buffer[0..1];
}
var i = usize(0);
pub fn encodeInto(out_stream: var, data: u64) !void {
var value = data;
while (value > 0) : (i += 1) {
buffer[i] = u8(0x80) + @truncate(u7, value);
value >>= 7;
while (value >= 0x80) : (value >>= 7) {
try out_stream.writeByte(u8(0x80) + @truncate(u7, value));
}
buffer[i - 1] &= 0x7F;
return buffer[0..i];
try out_stream.writeByte(@truncate(u7, value));
}

pub fn decode(bytes: []const u8, len: *usize) ParseError!u64 {
pub fn decodeFrom(in_stream: var) !u64 {
var value = u64(0);

for (bytes) |byte, i| {
if (i >= 10) {
return error.Overflow;
}
var i = usize(0);
while (i < 10) : (i += 1) {
const byte = try in_stream.readByte();
value += @intCast(u64, 0x7F & byte) << (7 * @intCast(u6, i));
if (byte & 0x80 == 0) {
len.* = i + 1;
return value;
}
}
// TODO: stream in bytes
return error.EndOfStream;

return error.Overflow;
}
};

test "Uint64Coder" {
var len: usize = 0;

var uint = try Uint64Coder.decode([_]u8{1}, &len);
var mem_stream = std.io.SliceInStream.init([_]u8{1});
var uint = try Uint64Coder.decodeFrom(&mem_stream.stream);
testing.expectEqual(u64(1), uint);

uint = try Uint64Coder.decode([_]u8{ 0b10101100, 0b00000010 }, &len);
mem_stream = std.io.SliceInStream.init([_]u8{ 0b10101100, 0b00000010 });
uint = try Uint64Coder.decodeFrom(&mem_stream.stream);
testing.expectEqual(u64(300), uint);

uint = try Uint64Coder.decode([_]u8{ 0b10010110, 0b00000001 }, &len);
mem_stream = std.io.SliceInStream.init([_]u8{ 0b10010110, 0b00000001 });
uint = try Uint64Coder.decodeFrom(&mem_stream.stream);
testing.expectEqual(u64(150), uint);
}

Expand All @@ -68,24 +54,24 @@ pub const Int64Coder = struct {
return Uint64Coder.encodeSize(@bitCast(u64, data));
}

pub fn encode(buffer: []u8, data: i64) []u8 {
return Uint64Coder.encode(buffer, @bitCast(u64, data));
pub fn encodeInto(out_stream: var, data: i64) !void {
try Uint64Coder.encodeInto(out_stream, @bitCast(u64, data));
}

pub fn decode(bytes: []const u8, len: *usize) ParseError!i64 {
return @bitCast(i64, try Uint64Coder.decode(bytes, len));
pub fn decodeFrom(in_stream: var) !i64 {
return @bitCast(i64, try Uint64Coder.decodeFrom(in_stream));
}
};

test "Int64Coder" {
var buf1: [1000]u8 = undefined;
var buf2: [1000]u8 = undefined;
var out1 = std.io.SliceOutStream.init(buf1[0..]);
var out2 = std.io.SliceOutStream.init(buf2[0..]);

testing.expectEqualSlices(
u8,
Uint64Coder.encode(buf1[0..], std.math.maxInt(u64)),
Int64Coder.encode(buf2[0..], -1),
);
try Uint64Coder.encodeInto(&out1.stream, std.math.maxInt(u64));
try Int64Coder.encodeInto(&out2.stream, -1);
testing.expectEqualSlices(u8, out1.getWritten(), out2.getWritten());
}

pub const Sint64Coder = struct {
Expand All @@ -95,12 +81,12 @@ pub const Sint64Coder = struct {
return Uint64Coder.encodeSize(@bitCast(u64, (data << 1) ^ (data >> 63)));
}

pub fn encode(buffer: []u8, data: i64) []u8 {
return Uint64Coder.encode(buffer, @bitCast(u64, (data << 1) ^ (data >> 63)));
pub fn encodeInto(out_stream: var, data: i64) !void {
try Uint64Coder.encodeInto(out_stream, @bitCast(u64, (data << 1) ^ (data >> 63)));
}

pub fn decode(bytes: []const u8, len: *usize) ParseError!i64 {
const source = try Uint64Coder.decode(bytes, len);
pub fn decodeFrom(in_stream: var) !i64 {
const source = try Uint64Coder.decodeFrom(in_stream);
const raw = @bitCast(i64, source >> 1);
return if (@mod(source, 2) == 0) raw else -(raw + 1);
}
Expand All @@ -109,24 +95,24 @@ pub const Sint64Coder = struct {
test "Sint64Coder" {
var buf1: [1000]u8 = undefined;
var buf2: [1000]u8 = undefined;

testing.expectEqualSlices(
u8,
Uint64Coder.encode(buf1[0..], 1),
Sint64Coder.encode(buf2[0..], -1),
);

testing.expectEqualSlices(
u8,
Uint64Coder.encode(buf1[0..], 4294967294),
Sint64Coder.encode(buf2[0..], 2147483647),
);

testing.expectEqualSlices(
u8,
Uint64Coder.encode(buf1[0..], 4294967295),
Sint64Coder.encode(buf2[0..], -2147483648),
);
var out1 = std.io.SliceOutStream.init(buf1[0..]);
var out2 = std.io.SliceOutStream.init(buf2[0..]);

try Uint64Coder.encodeInto(&out1.stream, 1);
try Sint64Coder.encodeInto(&out2.stream, -1);
testing.expectEqualSlices(u8, out1.getWritten(), out2.getWritten());

out1.reset();
out2.reset();
try Uint64Coder.encodeInto(&out1.stream, 4294967294);
try Sint64Coder.encodeInto(&out2.stream, 2147483647);
testing.expectEqualSlices(u8, out1.getWritten(), out2.getWritten());

out1.reset();
out2.reset();
try Uint64Coder.encodeInto(&out1.stream, 4294967295);
try Sint64Coder.encodeInto(&out2.stream, -2147483648);
testing.expectEqualSlices(u8, out1.getWritten(), out2.getWritten());
}

pub const Fixed64Coder = struct {
Expand All @@ -136,15 +122,14 @@ pub const Fixed64Coder = struct {
return 8;
}

pub fn encode(buffer: []u8, data: u64) []u8 {
var result = buffer[0..encodeSize(data)];
std.mem.writeIntSliceLittle(u64, result, data);
return result;
pub fn encodeInto(out_stream: var, data: u64) !void {
var buffer = [_]u8{0} ** 8;
std.mem.writeIntSliceLittle(u64, buffer[0..], data);
try out_stream.write(buffer[0..]);
}

pub fn decode(bytes: []const u8, len: *usize) ParseError!u64 {
len.* = 8;
return std.mem.readIntSliceLittle(u64, bytes);
pub fn decodeFrom(in_stream: var) !u64 {
return in_stream.readIntLittle(u64);
}
};

Expand All @@ -155,15 +140,14 @@ pub const Fixed32Coder = struct {
return 4;
}

pub fn encode(buffer: []u8, data: u32) []u8 {
var result = buffer[0..encodeSize(data)];
std.mem.writeIntSliceLittle(u32, result, data);
return result;
pub fn encodeInto(out_stream: var, data: u32) !void {
var buffer = [_]u8{0} ** 4;
std.mem.writeIntSliceLittle(u32, buffer[0..], data);
try out_stream.write(buffer[0..]);
}

pub fn decode(bytes: []const u8, len: *usize) ParseError!u32 {
len.* = 4;
return std.mem.readIntSliceLittle(u32, bytes);
pub fn decodeFrom(in_stream: var) !u32 {
return try in_stream.readIntLittle(u32);
}
};

Expand All @@ -173,34 +157,31 @@ pub const BytesCoder = struct {
return header_size + data.len;
}

pub fn encode(buffer: []u8, data: []const u8) []u8 {
const header = Uint64Coder.encode(buffer, data.len);
// TODO: use a generator instead of buffer overflow
std.mem.copy(u8, buffer[header.len..], data);
return buffer[0 .. header.len + data.len];
pub fn encodeInto(out_stream: var, data: []const u8) !void {
try Uint64Coder.encodeInto(out_stream, data.len);
try out_stream.write(data);
}

pub fn decode(buffer: []const u8, len: *usize, allocator: *std.mem.Allocator) ![]u8 {
var header_len: usize = undefined;
const header = try Uint64Coder.decode(buffer, &header_len);

var data = try allocator.alloc(u8, header);
errdefer allocator.free(data);
pub fn decodeFrom(in_stream: var, allocator: *std.mem.Allocator) ![]u8 {
const size = try Uint64Coder.decodeFrom(in_stream);

std.mem.copy(u8, data, buffer[header_len .. header_len + data.len]);
len.* = header_len + data.len;
var result = try allocator.alloc(u8, size);
errdefer allocator.free(result);

return data;
try in_stream.readNoEof(result);
return result;
}
};

test "BytesCoder" {
var buffer: [1000]u8 = undefined;
var buf: [1000]u8 = undefined;
var out = std.io.SliceOutStream.init(buf[0..]);
try BytesCoder.encodeInto(&out.stream, "testing");

testing.expectEqualSlices(
u8,
[_]u8{ 0x07, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67 },
BytesCoder.encode(buffer[0..], "testing"),
out.getWritten(),
);
}

Expand Down