forked from golang/protobuf
-
Notifications
You must be signed in to change notification settings - Fork 1
/
lib.go
380 lines (322 loc) · 11.2 KB
/
lib.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
// Go support for Protocol Buffers - Google's data interchange format
//
// Modifications copyright 2016 Mist Systems.
//
// Original code copyright 2010 The Go Authors. All rights reserved.
// https://github.com/golang/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
/*
Package protobuf3 encodes (marshals) Go structs which contain fields
tagged with 'protobuf' encoding information into protobuf version 3.
It can operate on the output generated by the protobuf compiler. However
it exists to operate on hand crafted types which are easier/more efficient
for the rest of the code to handle than the struct the protobuf compiler
emits would be.
*/
package protobuf3
import (
"errors"
"fmt"
"os"
"strings"
"sync"
"unsafe"
)
// Message is implemented by generated protocol buffer messages.
type Message interface {
// empty interface. As long as the fields are decorated with protobuf tags or the type implements Marshaler it's fine with us.
}
// A Buffer is a buffer manager for marshaling and unmarshaling
// protocol buffers. It may be reused between invocations to
// reduce memory usage. It is not necessary to use a Buffer;
// the global functions Marshal and Unmarshal create a
// temporary Buffer and are fine for most applications.
// However if you are decoding objects with large []bytes, creating
// your own Buffer and setting Immutable=true will result in the
// decode []bytes references directly into the []byte passed to NewBuffer()
// rather than being expensive copies.
type Buffer struct {
buf []byte // encode/decode byte stream
index uint // read position in .buf[]
Immutable bool // true if we the caller promises the contents of buf[] are immutable, and thus we can retain references to it for types which decode into []byte
err error // nil, or the first error which happened during operation
array_indexes map[unsafe.Pointer]uint // map of base address of array -> index of next unfilled slot (or nil if never used)
}
// NewBuffer allocates a new Buffer and initializes its internal data to
// the contents of the argument slice.
func NewBuffer(e []byte) *Buffer {
return &Buffer{buf: e}
}
// Reset resets the Buffer, ready for marshaling a new protocol buffer.
func (p *Buffer) Reset() {
p.buf = p.buf[0:0] // for reading/writing
p.index = 0 // for reading
p.err = nil
p.array_indexes = nil
}
var buffer_pool = sync.Pool{
New: func() interface{} { return NewBuffer(nil) },
}
// newBuffer gets a new buffer from our private buffer_pool
func newBuffer(e []byte) *Buffer {
p := buffer_pool.Get().(*Buffer)
p.buf = e
return p
}
// release returns the buffer to the pool and returns the now detached inner []byte
func (p *Buffer) release() []byte {
bytes := p.buf
p.buf = nil
p.index = 0
p.Immutable = false
p.err = nil
p.array_indexes = nil
buffer_pool.Put(p)
return bytes
}
// save the first error; toss the rest
// note: works correctly when arg err is nil
func (p *Buffer) noteError(err error) {
if p.err == nil {
p.err = err
}
}
// Bytes returns the contents of the Buffer.
func (p *Buffer) Bytes() []byte { return p.buf }
// Rewind resets the read point to the start of the buffer.
func (p *Buffer) Rewind() {
p.index = 0
}
// uint(len([]byte)), b/c using uint for indexing saves us some bounds checks for negative indexes
func ulen(buf []byte) uint { return uint(len(buf)) }
// returns true if we've decoded to the end of the buffer
func (p *Buffer) EOF() bool {
return ulen(p.buf) == p.index
}
// save our current position in decoding into an array
func (p *Buffer) saveIndex(ptr unsafe.Pointer, idx uint) {
if p.array_indexes == nil {
// the 1st time we need to allocate
p.array_indexes = make(map[unsafe.Pointer]uint)
}
p.array_indexes[ptr] = idx
}
// Next returns the next item in the message.
// Both the entire item and just the value bytes are returned. The entire item is useful because it is itself a valid protobuf message.
// If the end of the buffer is reached 0,nil,nil,0,nil is returned.
func (p *Buffer) Next() (id int, full []byte, val []byte, wt WireType, err error) {
start := p.index
if start == ulen(p.buf) {
// end of buffer
return 0, nil, nil, 0, nil
}
var vi uint64
vi, err = p.DecodeVarint()
if err != nil {
return 0, nil, nil, 0, err
}
wt = WireType(vi) & 7
id = int(vi >> 3)
val_start := p.index // correct except in the case of WireBytes, where the value starts after the varint byte length
switch wt {
case WireBytes:
val, err = p.DecodeRawBytes()
case WireVarint:
err = p.SkipVarint()
case WireFixed32:
err = p.SkipFixed(4)
case WireFixed64:
err = p.SkipFixed(8)
default:
err = fmt.Errorf("protobuf3: unsupported wiretype %d", int(wt))
}
if val == nil {
val = p.buf[val_start:p.index:p.index]
} // else val is already set up
return id, p.buf[start:p.index:p.index], val, wt, err
}
// Find scans forward for next item which has id 'id'.
// Both the entire item and just the value bytes are returned. The entire item is useful because it is itself a valid protobuf message.
// If no match is found, ErrNotFound is returned.
// If sorted is true then this function assumes the message's fields are sorted by id, and encountering any id > 'id' short circuits the search
func (p *Buffer) Find(id uint, sorted bool) (position int, full []byte, val []byte, wt WireType, err error) {
for p.index < ulen(p.buf) {
start := p.index
var vi uint64
vi, err = p.DecodeVarint()
if err != nil {
return 0, nil, nil, 0, err
}
wt = WireType(vi) & 7
if vi>>3 == uint64(id) {
// it's a match. size the value and return
val_start := p.index // correct except in the case of WireBytes, where the value starts after the varint byte length
switch wt {
case WireBytes:
val, err = p.DecodeRawBytes()
case WireVarint:
err = p.SkipVarint()
case WireFixed32:
err = p.SkipFixed(4)
case WireFixed64:
err = p.SkipFixed(8)
default:
err = fmt.Errorf("protobuf3: unsupported wiretype %d", int(wt))
}
if val == nil {
val = p.buf[val_start:p.index:p.index]
} // else val is already set up
return int(start), p.buf[start:p.index:p.index], val, wt, err
} else if sorted && vi>>3 > uint64(id) {
// we've advanced past the requested id, and we're assured the message is sorted by id
// so we can stop searching now
break
} else {
// skip over the ID's value
switch wt {
case WireBytes:
err = p.SkipRawBytes()
case WireVarint:
err = p.SkipVarint()
case WireFixed32:
err = p.SkipFixed(4)
case WireFixed64:
err = p.SkipFixed(8)
default:
err = fmt.Errorf("protobuf3: unsupported wiretype %d", int(wt))
}
if err != nil {
return 0, nil, nil, 0, err
}
}
}
// nothing found
return 0, nil, nil, 0, ErrNotFound
}
// FindBytes is similar to Find but only matches and returns ids with wiretype "bytes".
// If the id is present but hasn't got the wiretype "bytes" then ErrNotFound is returned.
// It exists because calling Find() and checking for WireBytes is a common pattern in calling code.
func (p *Buffer) FindBytes(id uint, sorted bool) (position int, full []byte, val []byte, err error) {
var wt WireType
position, full, val, wt, err = p.Find(id, sorted)
if err == nil && wt != WireBytes {
return 0, nil, nil, ErrNotFound
}
return position, full, val, err
}
// error returned by (*Buffer).Find when the id is not present in the buffer
var ErrNotFound = errors.New("ID not found in protobuf buffer")
// DebugPrint dumps the encoded data in b in a debugging format with a header
// including the string s. Used in testing but made available for general debugging.
func DebugPrint(b []byte) string {
var u uint64
p := NewBuffer(b)
depth := 0
var out strings.Builder
defer func() {
if x := recover(); x != nil {
fmt.Fprintln(os.Stderr, out.String())
panic(x)
}
}()
out:
for {
for i := 0; i < depth; i++ {
out.WriteString(" ")
}
index := p.index
if index == ulen(p.buf) {
break
}
op, err := p.DecodeVarint()
if err != nil {
out.WriteString(fmt.Sprintf("%3d: fetching op err %v\n", index, err))
break out
}
tag := op >> 3
wire := WireType(op) & 7
switch wire {
default:
out.WriteString(fmt.Sprintf("%3d: t=%3d, unknown wire=%d\n", index, tag, wire))
break out
case WireBytes:
var r []byte
r, err = p.DecodeRawBytes()
if err != nil {
break out
}
out.WriteString(fmt.Sprintf("%3d: t=%3d, bytes [%d]", index, tag, len(r)))
if len(r) <= 16 {
for i := 0; i < len(r); i++ {
out.WriteString(fmt.Sprintf(" %.2x", r[i]))
}
} else {
for i := 0; i < 4; i++ {
out.WriteString(fmt.Sprintf(" %.2x", r[i]))
}
out.WriteString(" ..")
for i := len(r) - 4; i < len(r); i++ {
out.WriteString(fmt.Sprintf(" %.2x", r[i]))
}
}
out.WriteString("\n")
case WireFixed32:
u, err = p.DecodeFixed32()
if err != nil {
out.WriteString(fmt.Sprintf("%3d: t=%3d, fix32 err %v\n", index, tag, err))
break out
}
out.WriteString(fmt.Sprintf("%3d: t=%3d, fix32 %d\n", index, tag, u))
case WireFixed64:
u, err = p.DecodeFixed64()
if err != nil {
out.WriteString(fmt.Sprintf("%3d: t=%3d, fix64 err %v\n", index, tag, err))
break out
}
out.WriteString(fmt.Sprintf("%3d: t=%3d, fix64 %d\n", index, tag, u))
case WireVarint:
u, err = p.DecodeVarint()
if err != nil {
out.WriteString(fmt.Sprintf("%3d: t=%3d, varint err %v\n", index, tag, err))
break out
}
out.WriteString(fmt.Sprintf("%3d: t=%3d, varint %d\n", index, tag, u))
case WireStartGroup:
out.WriteString(fmt.Sprintf("%3d: t=%3d, start\n", index, tag))
depth++
case WireEndGroup:
depth--
out.WriteString(fmt.Sprintf("%3d: t=%3d, end\n", index, tag))
}
}
if depth != 0 {
out.WriteString(fmt.Sprintf("%3d: start-end not balanced %d\n", p.index, depth))
}
out.WriteString("\n")
return out.String()
}