-
Notifications
You must be signed in to change notification settings - Fork 56
/
encode.jl
326 lines (286 loc) · 11.3 KB
/
encode.jl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
function encode_tag(io::IO, field_number, wire_type::WireType)
vbyte_encode(io, (UInt32(field_number) << 3) | UInt32(wire_type))
return nothing
end
encode_tag(e::ProtoEncoder, field_number, wire_type::WireType) = encode_tag(e.io, field_number, wire_type)
# TODO: audit usage and composability of maybe_ensure_room
maybe_ensure_room(io::IOBuffer, n) = Base.ensureroom(io, min(io.maxsize, n))
maybe_ensure_room(::IO, n) = nothing
@noinline _incomplete_encode_error(io::IOBuffer, nb, target) = throw(ArgumentError("Failed to write to IOBuffer, only written $nb bytes out of $target (maxsize: $(io.maxsize), positiom : $(position(io))"))
@noinline _incomplete_encode_error(::IO, nb, target) = throw(ArgumentError("Failed to write to IO, only written $nb bytes out of $target"))
@inline function _with_size(f, io::IOBuffer, sink, x, V...)
if io.seekable
initpos = position(io)
# We need to encode the encoded size of x before we know it. We first preallocate 1
# byte as that is the mininum size of the encoded size.
# If our guess is right, it will save us a copy, but we never want to preallocate too much
# space for the size, because then we risk outgrowing the buffer that was allocated with exact size
# needed to contain the message.
# TODO: make the guess better (e.g. by incorporating maxsize)
encoded_size_len_guess = 1
truncate(io, initpos + encoded_size_len_guess)
seek(io, initpos + encoded_size_len_guess)
# Now we can encode the object itself
f(sink, x, V...) # e.g. _encode(io, x) or _encode(io, x, Val{:zigzag})
endpos = position(io)
encoded_size = endpos - initpos - encoded_size_len_guess
encoded_size_len = _encoded_size(UInt32(encoded_size))
@assert (initpos + encoded_size_len + encoded_size) <= io.maxsize
# If our initial guess on encoded size of the size was wrong, then we need to move the encoded data
if encoded_size_len_guess < encoded_size_len
truncate(io, initpos + encoded_size_len + encoded_size)
# Move the data right after the correct size
unsafe_copyto!(
io.data,
initpos + encoded_size_len + 1,
io.data,
initpos + encoded_size_len_guess + 1,
encoded_size
)
end
# Now we can encode the size
seek(io, initpos)
vbyte_encode(io, UInt32(encoded_size))
seek(io, initpos + encoded_size_len + encoded_size)
else
# TODO: avoid quadratic behavior when estimating encoded size by providing a scratch buffer
vbyte_encode(io, UInt32(_encoded_size(x, V...)))
f(sink, x, V...)
end
return nothing
end
@inline function _with_size(f, io::IO, sink, x, V...)
# TODO: avoid quadratic behavior when estimating encoded size by providing a scratch buffer
vbyte_encode(io, UInt32(_encoded_size(x, V...)))
f(sink, x, V...)
return nothing
end
function _encode(io::IO, x::T) where {T<:Union{UInt32,UInt64}}
vbyte_encode(io, x)
return nothing
end
function _encode(io::IO, x::Int64)
vbyte_encode(io, reinterpret(UInt64, x))
return nothing
end
function _encode(io::IO, x::Int32)
x < 0 ? vbyte_encode(io, reinterpret(UInt64, Int64(x))) : vbyte_encode(io, reinterpret(UInt32, x))
return nothing
end
function _encode(io::IO, x::T) where {T<:Union{Enum{Int32},Enum{UInt32}}}
vbyte_encode(io, reinterpret(UInt32, x))
return nothing
end
function _encode(io::IO, x::Vector{T}, ::Type{Val{:fixed}}) where {T<:Union{Int32,Int64,UInt32,UInt64}}
nb = write(io, x)
nb == sizeof(x) || _incomplete_encode_error(io, nb, sizeof(x))
return nothing
end
function _encode(io::IO, x::T, ::Type{Val{:fixed}}) where {T<:Union{Int32,Int64,UInt32,UInt64}}
_unsafe_write(io, Ref(x), Core.sizeof(x))
return nothing
end
function _encode(io::IO, x::T, ::Type{Val{:zigzag}}) where {T<:Union{Int32,Int64}}
vbyte_encode(io, reinterpret(unsigned(T), zigzag_encode(x)))
return nothing
end
function _encode(io::IO, x::Vector{T}, ::Type{Val{:zigzag}}) where {T<:Union{Int32,Int64}}
maybe_ensure_room(io, length(x))
for el in x
_encode(io, el, Val{:zigzag})
end
return nothing
end
function _encode(io::IO, x::String)
nb = write(io, x)
nb == sizeof(x) || _incomplete_encode_error(io, nb, sizeof(x))
return nothing
end
function _encode(io::IO, x::T) where {T<:Union{Bool,Float32,Float64}}
_unsafe_write(io, Ref(x), Core.sizeof(x))
return nothing
end
function _encode(io::IO, x::Vector{T}) where {T<:Union{Bool,UInt8,Float32,Float64}}
nb = write(io, x)
nb == sizeof(x) || _incomplete_encode_error(io, nb, sizeof(x))
return nothing
end
function _encode(io::IO, x::Base.CodeUnits{UInt8, String})
nb = write(io, x)
nb == sizeof(x) || _incomplete_encode_error(io, nb, sizeof(x))
return nothing
end
function _encode(io::IO, x::Vector{T}) where {T<:Union{UInt32,UInt64,Int32,Int64,Enum{Int32},Enum{UInt32}}}
maybe_ensure_room(io, length(x))
for el in x
_encode(io, el)
end
return nothing
end
function encode(e::ProtoEncoder, i::Int, x::Dict{K,V}) where {K,V}
maybe_ensure_room(e.io, 2*(length(x)+1))
for (k, v) in x
# encode header for key-value pair message
encode_tag(e, i, LENGTH_DELIMITED)
vbyte_encode(e.io, UInt32(_encoded_size(k, 1) + _encoded_size(v, 2)))
encode(e, 1, k)
encode(e, 2, v)
end
nothing
end
for T in (:(:fixed), :(:zigzag))
@eval function encode(e::ProtoEncoder, i::Int, x::Dict{K,V}, ::Type{Val{Tuple{$(T),Nothing}}}) where {K,V}
maybe_ensure_room(e.io, 2*(length(x)+1))
for (k, v) in x
# encode header for key-value pair message
encode_tag(e, i, LENGTH_DELIMITED)
vbyte_encode(e.io, UInt32(_encoded_size(k, 1, Val{$(T)}) + _encoded_size(v, 2)))
encode(e, 1, k, Val{$(T)})
encode(e, 2, v)
end
nothing
end
@eval function encode(e::ProtoEncoder, i::Int, x::Dict{K,V}, ::Type{Val{Tuple{Nothing,$(T)}}}) where {K,V}
maybe_ensure_room(e.io, 2*(length(x)+1))
for (k, v) in x
# encode header for key-value pair message
encode_tag(e, i, LENGTH_DELIMITED)
vbyte_encode(e.io, UInt32(_encoded_size(k, 1) + _encoded_size(v, 2, Val{$(T)})))
encode(e, 1, k)
encode(e, 2, v, Val{$(T)})
end
nothing
end
end
for T in (:(:fixed), :(:zigzag)), S in (:(:fixed), :(:zigzag))
@eval function encode(e::AbstractProtoEncoder, i::Int, x::Dict{K,V}, ::Type{Val{Tuple{$(T),$(S)}}}) where {K,V}
maybe_ensure_room(e.io, 2*(length(x)+1))
for (k, v) in x
# encode header for key-value pair message
encode_tag(e, i, LENGTH_DELIMITED)
vbyte_encode(e.io, UInt32(_encoded_size(k, 1, Val{$(T)}) + _encoded_size(v, 2, Val{$(S)})))
encode(e, 1, k, Val{$(T)})
encode(e, 2, v, Val{$(S)})
end
nothing
end
end
function encode(e::AbstractProtoEncoder, i::Int, x::T) where {T<:Union{Bool,Int32,Int64,UInt32,UInt64,Enum{Int32},Enum{UInt32}}}
encode_tag(e, i, VARINT)
_encode(e.io, x)
return nothing
end
function encode(e::AbstractProtoEncoder, i::Int, x::T, ::Type{Val{:zigzag}}) where {T<:Union{Int32,Int64}}
encode_tag(e, i, VARINT)
_encode(e.io, x, Val{:zigzag})
return nothing
end
function encode(e::AbstractProtoEncoder, i::Int, x::T, ::Type{Val{:fixed}}) where {T<:Union{Int32,UInt32}}
encode_tag(e, i, FIXED32)
_encode(e.io, x, Val{:fixed})
return nothing
end
function encode(e::AbstractProtoEncoder, i::Int, x::Float32)
encode_tag(e, i, FIXED32)
_encode(e.io, x)
return nothing
end
function encode(e::AbstractProtoEncoder, i::Int, x::T, ::Type{Val{:fixed}}) where {T<:Union{Int64,UInt64}}
encode_tag(e, i, FIXED64)
_encode(e.io, x, Val{:fixed})
return nothing
end
function encode(e::AbstractProtoEncoder, i::Int, x::Float64)
encode_tag(e, i, FIXED64)
_encode(e.io, x)
return nothing
end
function encode(e::AbstractProtoEncoder, i::Int, x::Vector{T}) where {T<:Union{Bool,UInt8,Float32,Float64}}
encode_tag(e, i, LENGTH_DELIMITED)
vbyte_encode(e.io, UInt32(sizeof(x)))
_encode(e.io, x)
return nothing
end
function encode(e::AbstractProtoEncoder, i::Int, x::Base.CodeUnits{UInt8, String})
encode_tag(e, i, LENGTH_DELIMITED)
vbyte_encode(e.io, UInt32(sizeof(x)))
_encode(e.io, x)
return nothing
end
function encode(e::AbstractProtoEncoder, i::Int, x::Vector{String})
maybe_ensure_room(e.io, length(x) * (sizeof(first(x)) + 1))
for el in x
encode_tag(e, i, LENGTH_DELIMITED)
vbyte_encode(e.io, UInt32(sizeof(el)))
_encode(e.io, el)
end
return nothing
end
function encode(e::AbstractProtoEncoder, i::Int, x::Vector{Vector{UInt8}})
maybe_ensure_room(e.io, length(x) * (sizeof(first(x)) + 1))
for el in x
encode_tag(e, i, LENGTH_DELIMITED)
vbyte_encode(e.io, UInt32(sizeof(el)))
_encode(e.io, el)
end
return nothing
end
function encode(e::AbstractProtoEncoder, i::Int, x::String)
encode_tag(e, i, LENGTH_DELIMITED)
vbyte_encode(e.io, UInt32(sizeof(x)))
_encode(e.io, x)
return nothing
end
function encode(e::AbstractProtoEncoder, i::Int, x::Vector{T}, ::Type{Val{:fixed}}) where {T<:Union{UInt32,UInt64,Int32,Int64}}
encode_tag(e, i, LENGTH_DELIMITED)
vbyte_encode(e.io, UInt32(sizeof(x)))
_encode(e.io, x, Val{:fixed})
return nothing
end
function encode(e::AbstractProtoEncoder, i::Int, x::Vector{T}) where {T<:Union{UInt32,UInt64,Int32,Int64,Enum{Int32},Enum{UInt32}}}
encode_tag(e, i, LENGTH_DELIMITED)
_with_size(_encode, e.io, e.io, x)
return nothing
end
function encode(e::AbstractProtoEncoder, i::Int, x::Vector{T}, ::Type{Val{:zigzag}}) where {T<:Union{Int32,Int64}}
encode_tag(e, i, LENGTH_DELIMITED)
_with_size(_encode, e.io, e.io, x, Val{:zigzag})
return nothing
end
# T is a struct/message type
function encode(e::AbstractProtoEncoder, i::Int, x::Vector{T}) where {T}
@assert !isempty(x)
maybe_ensure_room(e.io, length(x) * (1 + sizeof(typeof(first(x)))))
for el in x
encode_tag(e, i, LENGTH_DELIMITED)
_with_size(encode, e.io, e, el)
end
return nothing
end
function encode(e::AbstractProtoEncoder, i::Int, x::T) where {T}
encode_tag(e, i, LENGTH_DELIMITED)
_with_size(encode, e.io, e, x)
return nothing
end
# Groups
function encode(e::AbstractProtoEncoder, i::Int, x::Vector{T}, ::Type{Val{:group}}) where {T}
@assert !isempty(x)
maybe_ensure_room(e.io, length(x) * (2 + sizeof(typeof(first(x)))))
for el in x
encode_tag(e, i, START_GROUP)
encode(e, el) # This method has to be generated by protojl
vbyte_encode(e.io, UInt32(END_GROUP))
end
return nothing
end
function encode(e::AbstractProtoEncoder, i::Int, x::T, ::Type{Val{:group}}) where {T}
maybe_ensure_room(e.io, 2 + sizeof(T))
encode_tag(e, i, START_GROUP)
encode(e, x) # This method has to be generated by protojl
vbyte_encode(e.io, UInt32(END_GROUP))
return nothing
end
# Resolving a method ambiguity
function encode(::AbstractProtoEncoder, ::Int, ::Dict{K, V}, ::Type{Val{:group}}) where {K, V}
throw(MethodError(encode, (AbstractProtoEncoder, Int, Dict{K, V}, Val{:group})))
end