From 1c5876990ef5c15245d8a3fff38fd6e027732355 Mon Sep 17 00:00:00 2001 From: Ugorji Nwoke Date: Thu, 20 Feb 2014 10:24:31 -0500 Subject: [PATCH] General refactoring, improve simple codec, improve tests and benchmarks. Fixes #30 --- codec/bench_test.go | 72 ++++++++--- codec/binc.go | 19 +-- codec/codecs_test.go | 95 +++++++------- codec/decode.go | 6 + codec/encode.go | 14 +- codec/helper.go | 15 +++ codec/helper_internal.go | 26 ++-- codec/msgpack.go | 14 +- codec/simple.go | 268 ++++++++++++++++++++------------------- 9 files changed, 297 insertions(+), 232 deletions(-) diff --git a/codec/bench_test.go b/codec/bench_test.go index dff5cc51..4d437035 100644 --- a/codec/bench_test.go +++ b/codec/bench_test.go @@ -62,7 +62,9 @@ func benchInit() { benchCheckers = append(benchCheckers, benchChecker{"msgpack", fnMsgpackEncodeFn, fnMsgpackDecodeFn}, - benchChecker{"binc", fnBincEncodeFn, fnBincDecodeFn}, + benchChecker{"binc-nosym", fnBincNoSymEncodeFn, fnBincNoSymDecodeFn}, + benchChecker{"binc-sym", fnBincSymEncodeFn, fnBincSymDecodeFn}, + benchChecker{"simple", fnSimpleEncodeFn, fnSimpleDecodeFn}, benchChecker{"gob", fnGobEncodeFn, fnGobDecodeFn}, benchChecker{"json", fnJsonEncodeFn, fnJsonDecodeFn}, ) @@ -74,7 +76,7 @@ func benchInit() { func runBenchInit() { logT(nil, "..............................................") logT(nil, "BENCHMARK INIT: %v", time.Now()) - logT(nil, "To run full benchmark comparing encodings (MsgPack, Binc, JSON, GOB, etc), "+ + logT(nil, "To run full benchmark comparing encodings (MsgPack, Binc, Simple, JSON, GOB, etc), "+ "use: \"go test -bench=.\"") logT(nil, "Benchmark: ") logT(nil, "\tStruct recursive Depth: %d", benchDepth) @@ -209,13 +211,45 @@ func fnMsgpackDecodeFn(buf []byte, ts interface{}) error { return NewDecoderBytes(buf, testMsgpackH).Decode(ts) } -func fnBincEncodeFn(ts interface{}) (bs []byte, err error) { +func fnBincEncodeFn(ts interface{}, sym AsSymbolFlag) (bs []byte, err error) { + tSym := testBincH.AsSymbols + testBincH.AsSymbols = sym err = NewEncoderBytes(&bs, testBincH).Encode(ts) + testBincH.AsSymbols = tSym + return +} + +func fnBincDecodeFn(buf []byte, ts interface{}, sym AsSymbolFlag) (err error) { + tSym := testBincH.AsSymbols + testBincH.AsSymbols = sym + err = NewDecoderBytes(buf, testBincH).Decode(ts) + testBincH.AsSymbols = tSym + return +} + +func fnBincNoSymEncodeFn(ts interface{}) (bs []byte, err error) { + return fnBincEncodeFn(ts, AsSymbolNone) +} + +func fnBincNoSymDecodeFn(buf []byte, ts interface{}) error { + return fnBincDecodeFn(buf, ts, AsSymbolNone) +} + +func fnBincSymEncodeFn(ts interface{}) (bs []byte, err error) { + return fnBincEncodeFn(ts, AsSymbolAll) +} + +func fnBincSymDecodeFn(buf []byte, ts interface{}) error { + return fnBincDecodeFn(buf, ts, AsSymbolAll) +} + +func fnSimpleEncodeFn(ts interface{}) (bs []byte, err error) { + err = NewEncoderBytes(&bs, testSimpleH).Encode(ts) return } -func fnBincDecodeFn(buf []byte, ts interface{}) error { - return NewDecoderBytes(buf, testBincH).Decode(ts) +func fnSimpleDecodeFn(buf []byte, ts interface{}) error { + return NewDecoderBytes(buf, testSimpleH).Decode(ts) } func fnGobEncodeFn(ts interface{}) ([]byte, error) { @@ -245,31 +279,27 @@ func Benchmark__Msgpack____Decode(b *testing.B) { } func Benchmark__Binc_NoSym_Encode(b *testing.B) { - tSym := testBincH.AsSymbols - testBincH.AsSymbols = AsSymbolNone - fnBenchmarkEncode(b, "binc", benchTs, fnBincEncodeFn) - testBincH.AsSymbols = tSym + fnBenchmarkEncode(b, "binc", benchTs, fnBincNoSymEncodeFn) } func Benchmark__Binc_NoSym_Decode(b *testing.B) { - tSym := testBincH.AsSymbols - testBincH.AsSymbols = AsSymbolNone - fnBenchmarkDecode(b, "binc", benchTs, fnBincEncodeFn, fnBincDecodeFn, fnBenchNewTs) - testBincH.AsSymbols = tSym + fnBenchmarkDecode(b, "binc", benchTs, fnBincNoSymEncodeFn, fnBincNoSymDecodeFn, fnBenchNewTs) } func Benchmark__Binc_Sym___Encode(b *testing.B) { - tSym := testBincH.AsSymbols - testBincH.AsSymbols = AsSymbolAll - fnBenchmarkEncode(b, "binc", benchTs, fnBincEncodeFn) - testBincH.AsSymbols = tSym + fnBenchmarkEncode(b, "binc", benchTs, fnBincSymEncodeFn) } func Benchmark__Binc_Sym___Decode(b *testing.B) { - tSym := testBincH.AsSymbols - testBincH.AsSymbols = AsSymbolAll - fnBenchmarkDecode(b, "binc", benchTs, fnBincEncodeFn, fnBincDecodeFn, fnBenchNewTs) - testBincH.AsSymbols = tSym + fnBenchmarkDecode(b, "binc", benchTs, fnBincSymEncodeFn, fnBincSymDecodeFn, fnBenchNewTs) +} + +func Benchmark__Simple____Encode(b *testing.B) { + fnBenchmarkEncode(b, "simple", benchTs, fnSimpleEncodeFn) +} + +func Benchmark__Simple____Decode(b *testing.B) { + fnBenchmarkDecode(b, "simple", benchTs, fnSimpleEncodeFn, fnSimpleDecodeFn, fnBenchNewTs) } func Benchmark__Gob________Encode(b *testing.B) { diff --git a/codec/binc.go b/codec/binc.go index 104a928a..2bb5e8fe 100644 --- a/codec/binc.go +++ b/codec/binc.go @@ -123,14 +123,13 @@ func (e *bincEncDriver) encodeFloat64(f float64) { } func (e *bincEncDriver) encIntegerPrune(bd byte, pos bool, v uint64, lim uint8) { - eb := e.b[:lim] if lim == 4 { - bigen.PutUint32(eb, uint32(v)) + bigen.PutUint32(e.b[:lim], uint32(v)) } else { - bigen.PutUint64(eb, v) + bigen.PutUint64(e.b[:lim], v) } if bincDoPrune { - i := pruneSignExt(eb, pos) + i := pruneSignExt(e.b[:lim], pos) e.w.writen1(bd | lim - 1 - byte(i)) e.w.writeb(e.b[i:lim]) } else { @@ -530,17 +529,7 @@ func (d *bincDecDriver) decodeFloat(chkOverflow32 bool) (f float64) { _, i, _ := d.decIntAny() f = float64(i) } - - // check overflow (logic adapted from std pkg reflect/value.go OverflowFloat() - if chkOverflow32 { - f2 := f - if f2 < 0 { - f2 = -f - } - if math.MaxFloat32 < f2 && f2 <= math.MaxFloat64 { - decErr("Overflow float32 value: %v", f2) - } - } + checkOverflowFloat32(f, chkOverflow32) d.bdRead = false return } diff --git a/codec/codecs_test.go b/codec/codecs_test.go index 517b24f4..cb184491 100644 --- a/codec/codecs_test.go +++ b/codec/codecs_test.go @@ -72,6 +72,7 @@ var ( testRpcInt = new(TestRpcInt) testMsgpackH = &MsgpackHandle{} testBincH = &BincHandle{} + testSimpleH = &SimpleHandle{} ) func testInitFlags() { @@ -277,18 +278,20 @@ func testInit() { testMsgpackH.RawToString = true // testMsgpackH.AddExt(byteSliceTyp, 0, testMsgpackH.BinaryEncodeExt, testMsgpackH.BinaryDecodeExt) // testMsgpackH.AddExt(timeTyp, 1, testMsgpackH.TimeEncodeExt, testMsgpackH.TimeDecodeExt) - testMsgpackH.AddExt(timeTyp, 1, - func(rv reflect.Value) ([]byte, error) { - return encodeTime(rv.Interface().(time.Time)), nil - }, - func(rv reflect.Value, bs []byte) error { - tt, err := decodeTime(bs) - if err == nil { - rv.Set(reflect.ValueOf(tt)) - } - return err - }, - ) + timeEncExt := func(rv reflect.Value) ([]byte, error) { + return encodeTime(rv.Interface().(time.Time)), nil + } + timeDecExt := func(rv reflect.Value, bs []byte) error { + tt, err := decodeTime(bs) + if err == nil { + rv.Set(reflect.ValueOf(tt)) + } + return err + } + + // add extensions for msgpack, simple for time.Time, so we can encode/decode same way. + testMsgpackH.AddExt(timeTyp, 1, timeEncExt, timeDecExt) + testSimpleH.AddExt(timeTyp, 1, timeEncExt, timeDecExt) primitives := []interface{}{ int8(-8), @@ -561,20 +564,18 @@ func doTestCodecTableOne(t *testing.T, testNil bool, h Handle, func testCodecTableOne(t *testing.T, h Handle) { // func TestMsgpackAllExperimental(t *testing.T) { // dopts := testDecOpts(nil, nil, false, true, true), - var oldWriteExt, oldRawToString bool + switch v := h.(type) { case *MsgpackHandle: + var oldWriteExt, oldRawToString bool oldWriteExt, v.WriteExt = v.WriteExt, true oldRawToString, v.RawToString = v.RawToString, true - } - doTestCodecTableOne(t, false, h, table, tableVerify) - //if true { panic("") } - switch v := h.(type) { - case *MsgpackHandle: + doTestCodecTableOne(t, false, h, table, tableVerify) v.WriteExt, v.RawToString = oldWriteExt, oldRawToString + default: + doTestCodecTableOne(t, false, h, table, tableVerify) } // func TestMsgpackAll(t *testing.T) { - idxTime, numPrim, numMap := 19, 23, 4 //skip []interface{} containing time.Time @@ -582,22 +583,14 @@ func testCodecTableOne(t *testing.T, h Handle) { doTestCodecTableOne(t, false, h, table[numPrim+1:], tableVerify[numPrim+1:]) // func TestMsgpackNilStringMap(t *testing.T) { var oldMapType reflect.Type - switch v := h.(type) { - case *MsgpackHandle: - oldMapType, v.MapType = v.MapType, mapStrIntfTyp - case *BincHandle: - oldMapType, v.MapType = v.MapType, mapStrIntfTyp - } + v := h.getBasicHandle() + oldMapType, v.MapType = v.MapType, mapStrIntfTyp + //skip time.Time, []interface{} containing time.Time, last map, and newStruc doTestCodecTableOne(t, true, h, table[:idxTime], tableTestNilVerify[:idxTime]) doTestCodecTableOne(t, true, h, table[numPrim+1:numPrim+numMap], tableTestNilVerify[numPrim+1:numPrim+numMap]) - switch v := h.(type) { - case *MsgpackHandle: - v.MapType = oldMapType - case *BincHandle: - v.MapType = oldMapType - } + v.MapType = oldMapType // func TestMsgpackNilIntf(t *testing.T) { @@ -951,6 +944,30 @@ func doTestMsgpackRpcSpecPythonClientToGoSvc(t *testing.T) { fmt.Sprintf("%#v\n%#v\n", []string{"A1", "B2", "C3"}, TestABC{"Aa", "Bb", "Cc"}), "cmdout=") } +func TestBincCodecsTable(t *testing.T) { + testCodecTableOne(t, testBincH) +} + +func TestBincCodecsMisc(t *testing.T) { + testCodecMiscOne(t, testBincH) +} + +func TestBincCodecsEmbeddedPointer(t *testing.T) { + testCodecEmbeddedPointer(t, testBincH) +} + +func TestSimpleCodecsTable(t *testing.T) { + testCodecTableOne(t, testSimpleH) +} + +func TestSimpleCodecsMisc(t *testing.T) { + testCodecMiscOne(t, testSimpleH) +} + +func TestSimpleCodecsEmbeddedPointer(t *testing.T) { + testCodecEmbeddedPointer(t, testSimpleH) +} + func TestMsgpackCodecsTable(t *testing.T) { testCodecTableOne(t, testMsgpackH) } @@ -963,16 +980,12 @@ func TestMsgpackCodecsEmbeddedPointer(t *testing.T) { testCodecEmbeddedPointer(t, testMsgpackH) } -func TestBincCodecsTable(t *testing.T) { - testCodecTableOne(t, testBincH) -} - -func TestBincCodecsMisc(t *testing.T) { - testCodecMiscOne(t, testBincH) +func TestBincRpcGo(t *testing.T) { + doTestRpcOne(t, GoRpc, testBincH, true, 0) } -func TestBincCodecsEmbeddedPointer(t *testing.T) { - testCodecEmbeddedPointer(t, testBincH) +func _TestSimpleRpcGo(t *testing.T) { + doTestRpcOne(t, GoRpc, testSimpleH, true, 0) } func TestMsgpackRpcGo(t *testing.T) { @@ -983,10 +996,6 @@ func TestMsgpackRpcSpec(t *testing.T) { doTestRpcOne(t, MsgpackSpecRpc, testMsgpackH, true, 0) } -func TestBincRpcGo(t *testing.T) { - doTestRpcOne(t, GoRpc, testBincH, true, 0) -} - // TODO: // Add Tests for: // - decoding empty list/map in stream into a nil slice/map diff --git a/codec/decode.go b/codec/decode.go index 19f16197..87bef2b9 100644 --- a/codec/decode.go +++ b/codec/decode.go @@ -69,6 +69,9 @@ type ioDecReader struct { } func (z *ioDecReader) readn(n int) (bs []byte) { + if n <= 0 { + return + } bs = make([]byte, n) if _, err := io.ReadAtLeast(z.r, bs, n); err != nil { panic(err) @@ -133,6 +136,9 @@ func (z *bytesDecReader) consume(n int) (oldcursor int) { } func (z *bytesDecReader) readn(n int) (bs []byte) { + if n <= 0 { + return + } c0 := z.consume(n) bs = z.b[c0:z.c] return diff --git a/codec/encode.go b/codec/encode.go index 73461f76..4914be0c 100644 --- a/codec/encode.go +++ b/codec/encode.go @@ -151,6 +151,9 @@ func (z *ioEncWriter) writeUint64(v uint64) { } func (z *ioEncWriter) writeb(bs []byte) { + if len(bs) == 0 { + return + } n, err := z.w.Write(bs) if err != nil { panic(err) @@ -220,6 +223,9 @@ func (z *bytesEncWriter) writeUint64(v uint64) { } func (z *bytesEncWriter) writeb(s []byte) { + if len(s) == 0 { + return + } c := z.grow(len(s)) copy(z.b[c:], s) } @@ -385,7 +391,7 @@ func (f *encFnInfo) kSlice(rv reflect.Value) { f.ee.encodeStringBytes(c_RAW, rv.Bytes()) return } - + l := rv.Len() if f.ti.mbs { if l%2 == 1 { @@ -410,7 +416,7 @@ func (f *encFnInfo) kArray(rv reflect.Value) { // So we have to duplicate the functionality here. // f.e.encodeValue(rv.Slice(0, rv.Len())) // f.kSlice(rv.Slice(0, rv.Len())) - + l := rv.Len() // Handle an array of bytes specially (in line with what is done for slices) if f.ti.rt.Elem().Kind() == reflect.Uint8 { @@ -418,7 +424,7 @@ func (f *encFnInfo) kArray(rv reflect.Value) { f.ee.encodeStringBytes(c_RAW, nil) return } - var bs []byte + var bs []byte if rv.CanAddr() { bs = rv.Slice(0, l).Bytes() } else { @@ -430,7 +436,7 @@ func (f *encFnInfo) kArray(rv reflect.Value) { f.ee.encodeStringBytes(c_RAW, bs) return } - + if f.ti.mbs { if l%2 == 1 { encErr("mapBySlice: invalid length (must be divisible by 2): %v", l) diff --git a/codec/helper.go b/codec/helper.go index 7f5a1588..e6dc0563 100644 --- a/codec/helper.go +++ b/codec/helper.go @@ -8,6 +8,7 @@ package codec import ( "encoding/binary" "fmt" + "math" "reflect" "sort" "strings" @@ -556,6 +557,20 @@ func doPanic(tag string, format string, params ...interface{}) { panic(fmt.Errorf("%s: "+format, params2...)) } +func checkOverflowFloat32(f float64, doCheck bool) { + if !doCheck { + return + } + // check overflow (logic adapted from std pkg reflect/value.go OverflowFloat() + f2 := f + if f2 < 0 { + f2 = -f + } + if math.MaxFloat32 < f2 && f2 <= math.MaxFloat64 { + decErr("Overflow float32 value: %v", f2) + } +} + func checkOverflow(ui uint64, i int64, bitsize uint8) { // check overflow (logic adapted from std pkg reflect/value.go OverflowUint() if bitsize == 0 { diff --git a/codec/helper_internal.go b/codec/helper_internal.go index 639ba137..58417da9 100644 --- a/codec/helper_internal.go +++ b/codec/helper_internal.go @@ -33,8 +33,7 @@ func panicValToErr(panicVal interface{}, err *error) { return } -func isEmptyValue(v reflect.Value) bool { - const deref = true +func isEmptyValueDeref(v reflect.Value, deref bool) bool { switch v.Kind() { case reflect.Array, reflect.Map, reflect.Slice, reflect.String: return v.Len() == 0 @@ -51,23 +50,30 @@ func isEmptyValue(v reflect.Value) bool { if v.IsNil() { return true } - return isEmptyValue(v.Elem()) + return isEmptyValueDeref(v.Elem(), deref) } else { return v.IsNil() } case reflect.Struct: // return true if all fields are empty. else return false. - return v.Interface() == reflect.Zero(v.Type()).Interface() - // for i, n := 0, v.NumField(); i < n; i++ { - // if !isEmptyValue(v.Field(i), deref) { - // return false - // } - // } - // return true + + // we cannot use equality check, because some fields may be maps/slices/etc + // and consequently the structs are not comparable. + // return v.Interface() == reflect.Zero(v.Type()).Interface() + for i, n := 0, v.NumField(); i < n; i++ { + if !isEmptyValueDeref(v.Field(i), deref) { + return false + } + } + return true } return false } +func isEmptyValue(v reflect.Value) bool { + return isEmptyValueDeref(v, true) +} + func debugf(format string, args ...interface{}) { if debugging { if len(format) == 0 || format[len(format)-1] != '\n' { diff --git a/codec/msgpack.go b/codec/msgpack.go index 21276186..da0500d1 100644 --- a/codec/msgpack.go +++ b/codec/msgpack.go @@ -472,16 +472,7 @@ func (d *msgpackDecDriver) decodeFloat(chkOverflow32 bool) (f float64) { default: f = float64(d.decodeInt(0)) } - // check overflow (logic adapted from std pkg reflect/value.go OverflowFloat() - if chkOverflow32 { - f2 := f - if f2 < 0 { - f2 = -f - } - if math.MaxFloat32 < f2 && f2 <= math.MaxFloat64 { - decErr("Overflow float32 value: %v", f2) - } - } + checkOverflowFloat32(f, chkOverflow32) d.bdRead = false return } @@ -820,3 +811,6 @@ func (x msgpackSpecRpc) ServerCodec(conn io.ReadWriteCloser, h Handle) rpc.Serve func (x msgpackSpecRpc) ClientCodec(conn io.ReadWriteCloser, h Handle) rpc.ClientCodec { return &msgpackSpecRpcCodec{newRPCCodec(conn, h)} } + +var _ decDriver = (*msgpackDecDriver)(nil) +var _ encDriver = (*msgpackEncDriver)(nil) diff --git a/codec/simple.go b/codec/simple.go index 8ce579b8..7cdd7e61 100644 --- a/codec/simple.go +++ b/codec/simple.go @@ -1,55 +1,34 @@ // Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved. // Use of this source code is governed by a BSD-style license found in the LICENSE file. -/* -simple is a simplistic codec which does not pack integers, floats, etc into smaller chunks. - -All numbers are encoded with 8 bytes (int64, uint64, float64). - -Beyond this, it looks very similar to binc or msgpack. - -- nil, false, true are represented with single bytes -- positive/negative ints, and floats are distinct types, all represented by 1,2,4 or 8 bytes -- lengths of containers (strings, arrays, ext, map) occupy 2 bytes and - exist right after the bd -- maps are encoded as [bd] [length] [[key][value]]... -- arrays are encoded as [bd] [length] [value]... -- extensions are encoded as [bd] [length] [tag] [byte]... -- strings/bytearrays are encoded as [bd] [length] [tag] [byte]... - -*/ - package codec import "math" const ( - _ byte = iota - simpleVdNil - simpleVdFalse - simpleVdTrue - simpleVdPosInt1 - simpleVdPosInt2 - simpleVdPosInt4 - simpleVdPosInt8 - simpleVdNegInt1 - simpleVdNegInt2 - simpleVdNegInt4 - simpleVdNegInt8 - simpleVdFloat32 - simpleVdFloat64 - - simpleVdString - simpleVdByteArray - simpleVdArray - simpleVdMap - - simpleVdExt + _ uint8 = iota + simpleVdNil = 1 + simpleVdFalse = 2 + simpleVdTrue = 3 + simpleVdFloat32 = 4 + simpleVdFloat64 = 5 + + // each lasts for 4 (ie n, n+1, n+2, n+3) + simpleVdPosInt = 8 + simpleVdNegInt = 12 + + // containers: each lasts for 4 (ie n, n+1, n+2, ... n+7) + simpleVdString = 216 + simpleVdByteArray = 224 + simpleVdArray = 232 + simpleVdMap = 240 + simpleVdExt = 248 ) type simpleEncDriver struct { + h *SimpleHandle w encWriter - b [8]byte + //b [8]byte } func (e *simpleEncDriver) isBuiltinType(rt uintptr) bool { @@ -61,7 +40,6 @@ func (e *simpleEncDriver) encodeBuiltin(rt uintptr, v interface{}) { func (e *simpleEncDriver) encodeNil() { e.w.writen1(simpleVdNil) - // e.w.writen1(0) } func (e *simpleEncDriver) encodeBool(b bool) { @@ -84,68 +62,66 @@ func (e *simpleEncDriver) encodeFloat64(f float64) { func (e *simpleEncDriver) encodeInt(v int64) { if v < 0 { - e.encUint(uint64(-v), false) + e.encUint(uint64(-v), simpleVdNegInt) } else { - e.encUint(uint64(v), true) + e.encUint(uint64(v), simpleVdPosInt) } } func (e *simpleEncDriver) encodeUint(v uint64) { - e.encUint(v, true) + e.encUint(v, simpleVdPosInt) } -func (e *simpleEncDriver) encUint(v uint64, pos bool) { +func (e *simpleEncDriver) encUint(v uint64, bd uint8) { switch { case v <= math.MaxUint8: - if pos { - e.w.writen1(simpleVdPosInt1) - } else { - e.w.writen1(simpleVdNegInt1) - } - e.w.writen1(uint8(v)) + e.w.writen2(bd, uint8(v)) case v <= math.MaxUint16: - if pos { - e.w.writen1(simpleVdPosInt2) - } else { - e.w.writen1(simpleVdNegInt2) - } + e.w.writen1(bd + 1) e.w.writeUint16(uint16(v)) case v <= math.MaxUint32: - if pos { - e.w.writen1(simpleVdPosInt4) - } else { - e.w.writen1(simpleVdNegInt4) - } + e.w.writen1(bd + 2) e.w.writeUint32(uint32(v)) case v <= math.MaxUint64: - if pos { - e.w.writen1(simpleVdPosInt8) - } else { - e.w.writen1(simpleVdNegInt8) - } + e.w.writen1(bd + 3) e.w.writeUint64(v) } } +func (e *simpleEncDriver) encLen(bd byte, length int) { + switch { + case length == 0: + e.w.writen1(bd) + case length <= math.MaxUint8: + e.w.writen1(bd + 1) + e.w.writen1(uint8(length)) + case length <= math.MaxUint16: + e.w.writen1(bd + 2) + e.w.writeUint16(uint16(length)) + case length <= math.MaxUint32: + e.w.writen1(bd + 3) + e.w.writeUint32(uint32(length)) + default: + e.w.writen1(bd + 4) + e.w.writeUint64(uint64(length)) + } +} + func (e *simpleEncDriver) encodeExtPreamble(xtag byte, length int) { - e.w.writen1(simpleVdExt) - e.w.writeUint32(uint32(length)) + e.encLen(simpleVdExt, length) e.w.writen1(xtag) } func (e *simpleEncDriver) encodeArrayPreamble(length int) { - e.w.writen1(simpleVdArray) - e.w.writeUint32(uint32(length)) + e.encLen(simpleVdArray, length) } func (e *simpleEncDriver) encodeMapPreamble(length int) { - e.w.writen1(simpleVdMap) - e.w.writeUint32(uint32(length)) + e.encLen(simpleVdMap, length) } func (e *simpleEncDriver) encodeString(c charEncoding, v string) { - e.w.writen1(simpleVdString) - e.w.writeUint32(uint32(len(v))) + e.encLen(simpleVdString, len(v)) e.w.writestr(v) } @@ -154,19 +130,19 @@ func (e *simpleEncDriver) encodeSymbol(v string) { } func (e *simpleEncDriver) encodeStringBytes(c charEncoding, v []byte) { - e.w.writen1(simpleVdByteArray) - e.w.writeUint32(uint32(len(v))) + e.encLen(simpleVdByteArray, len(v)) e.w.writeb(v) } //------------------------------------ type simpleDecDriver struct { + h *SimpleHandle r decReader bdRead bool bdType valueType bd byte - b [8]byte + //b [8]byte } func (d *simpleDecDriver) initReadNext() { @@ -185,21 +161,21 @@ func (d *simpleDecDriver) currentEncodedType() valueType { d.bdType = valueTypeNil case simpleVdTrue, simpleVdFalse: d.bdType = valueTypeBool - case simpleVdPosInt1, simpleVdPosInt2, simpleVdPosInt4, simpleVdPosInt8: + case simpleVdPosInt, simpleVdPosInt + 1, simpleVdPosInt + 2, simpleVdPosInt + 3: d.bdType = valueTypeUint - case simpleVdNegInt1, simpleVdNegInt2, simpleVdNegInt4, simpleVdNegInt8: + case simpleVdNegInt, simpleVdNegInt + 1, simpleVdNegInt + 2, simpleVdNegInt + 3: d.bdType = valueTypeInt case simpleVdFloat32, simpleVdFloat64: d.bdType = valueTypeFloat - case simpleVdString: + case simpleVdString, simpleVdString + 1, simpleVdString + 2, simpleVdString + 3, simpleVdString + 4: d.bdType = valueTypeString - case simpleVdByteArray: + case simpleVdByteArray, simpleVdByteArray + 1, simpleVdByteArray + 2, simpleVdByteArray + 3, simpleVdByteArray + 4: d.bdType = valueTypeBytes - case simpleVdExt: + case simpleVdExt, simpleVdExt + 1, simpleVdExt + 2, simpleVdExt + 3, simpleVdExt + 4: d.bdType = valueTypeExt - case simpleVdArray: + case simpleVdArray, simpleVdArray + 1, simpleVdArray + 2, simpleVdArray + 3, simpleVdArray + 4: d.bdType = valueTypeArray - case simpleVdMap: + case simpleVdMap, simpleVdMap + 1, simpleVdMap + 2, simpleVdMap + 3, simpleVdMap + 4: d.bdType = valueTypeMap default: decErr("currentEncodedType: Unrecognized d.vd: 0x%x", d.bd) @@ -225,34 +201,41 @@ func (d *simpleDecDriver) decodeBuiltin(rt uintptr, v interface{}) { func (d *simpleDecDriver) decIntAny() (ui uint64, i int64, neg bool) { switch d.bd { - case simpleVdPosInt1: + case simpleVdPosInt: ui = uint64(d.r.readn1()) - case simpleVdPosInt2: + i = int64(ui) + case simpleVdPosInt + 1: ui = uint64(d.r.readUint16()) - case simpleVdPosInt4: + i = int64(ui) + case simpleVdPosInt + 2: ui = uint64(d.r.readUint32()) - case simpleVdPosInt8: + i = int64(ui) + case simpleVdPosInt + 3: ui = uint64(d.r.readUint64()) - case simpleVdNegInt1: + i = int64(ui) + case simpleVdNegInt: ui = uint64(d.r.readn1()) + i = -(int64(ui)) neg = true - case simpleVdNegInt2: + case simpleVdNegInt + 1: ui = uint64(d.r.readUint16()) + i = -(int64(ui)) neg = true - case simpleVdNegInt4: + case simpleVdNegInt + 2: ui = uint64(d.r.readUint32()) + i = -(int64(ui)) neg = true - case simpleVdNegInt8: + case simpleVdNegInt + 3: ui = uint64(d.r.readUint64()) + i = -(int64(ui)) neg = true default: - decErr("Integer only valid from pos/neg integer1..8. Invalid descriptor: %v", d.bd) - } - if neg { - i = -(int64(ui)) - } else { - i = int64(ui) + decErr("decIntAny: Integer only valid from pos/neg integer1..8. Invalid descriptor: %v", d.bd) } + // don't do this check, because callers may only want the unsigned value. + // if ui > math.MaxInt64 { + // decErr("decIntAny: Integer out of range for signed int64: %v", ui) + // } return } @@ -280,44 +263,60 @@ func (d *simpleDecDriver) decodeFloat(chkOverflow32 bool) (f float64) { case simpleVdFloat64: f = math.Float64frombits(d.r.readUint64()) default: - if d.bd >= simpleVdPosInt1 && d.bd <= simpleVdNegInt8 { + if d.bd >= simpleVdPosInt && d.bd <= simpleVdNegInt+3 { _, i, _ := d.decIntAny() f = float64(i) } else { decErr("Float only valid from float32/64: Invalid descriptor: %v", d.bd) } } - // check overflow (logic adapted from std pkg reflect/value.go OverflowFloat() - if chkOverflow32 { - f2 := f - if f2 < 0 { - f2 = -f - } - if math.MaxFloat32 < f2 && f2 <= math.MaxFloat64 { - decErr("Overflow float32 value: %v", f2) - } - } + checkOverflowFloat32(f, chkOverflow32) + d.bdRead = false return } // bool can be decoded from bool only (single byte). func (d *simpleDecDriver) decodeBool() (b bool) { - b = d.r.readn1() != 0 + switch d.bd { + case simpleVdTrue: + b = true + case simpleVdFalse: + default: + decErr("Invalid single-byte value for bool: %s: %x", msgBadDesc, d.bd) + } d.bdRead = false return } func (d *simpleDecDriver) readMapLen() (length int) { + d.bdRead = false return d.decLen() } func (d *simpleDecDriver) readArrayLen() (length int) { + d.bdRead = false return d.decLen() } func (d *simpleDecDriver) decLen() int { - d.r.readb(d.b[:4]) - return int(bigen.Uint32(d.b[:4])) + switch d.bd % 8 { + case 0: + return 0 + case 1: + return int(d.r.readn1()) + case 2: + return int(d.r.readUint16()) + case 3: + ui := uint64(d.r.readUint32()) + checkOverflow(ui, 0, intBitsize) + return int(ui) + case 4: + ui := d.r.readUint64() + checkOverflow(ui, 0, intBitsize) + return int(ui) + } + decErr("decLen: Cannot read length: bd%8 must be in range 0..4. Got: %d", d.bd%8) + return -1 } func (d *simpleDecDriver) decodeString() (s string) { @@ -346,14 +345,14 @@ func (d *simpleDecDriver) decodeBytes(bs []byte) (bsOut []byte, changed bool) { func (d *simpleDecDriver) decodeExt(verifyTag bool, tag byte) (xtag byte, xbs []byte) { switch d.bd { - case simpleVdExt: + case simpleVdExt, simpleVdExt + 1, simpleVdExt + 2, simpleVdExt + 3, simpleVdExt + 4: l := d.decLen() xtag = d.r.readn1() if verifyTag && xtag != tag { decErr("Wrong extension tag. Got %b. Expecting: %v", xtag, tag) } xbs = d.r.readn(l) - case simpleVdByteArray: + case simpleVdByteArray, simpleVdByteArray + 1, simpleVdByteArray + 2, simpleVdByteArray + 3, simpleVdByteArray + 4: xbs, _ = d.decodeBytes(nil) default: decErr("Invalid d.vd for extensions (Expecting extensions or byte array). Got: 0x%x", d.bd) @@ -374,11 +373,11 @@ func (d *simpleDecDriver) decodeNaked() (v interface{}, vt valueType, decodeFurt case simpleVdTrue: vt = valueTypeBool v = true - case simpleVdPosInt1, simpleVdPosInt2, simpleVdPosInt4, simpleVdPosInt8: + case simpleVdPosInt, simpleVdPosInt + 1, simpleVdPosInt + 2, simpleVdPosInt + 3: vt = valueTypeUint ui, _, _ := d.decIntAny() v = ui - case simpleVdNegInt1, simpleVdNegInt2, simpleVdNegInt4, simpleVdNegInt8: + case simpleVdNegInt, simpleVdNegInt + 1, simpleVdNegInt + 2, simpleVdNegInt + 3: vt = valueTypeInt _, i, _ := d.decIntAny() v = i @@ -388,13 +387,13 @@ func (d *simpleDecDriver) decodeNaked() (v interface{}, vt valueType, decodeFurt case simpleVdFloat64: vt = valueTypeFloat v = d.decodeFloat(false) - case simpleVdString: + case simpleVdString, simpleVdString + 1, simpleVdString + 2, simpleVdString + 3, simpleVdString + 4: vt = valueTypeString v = d.decodeString() - case simpleVdByteArray: + case simpleVdByteArray, simpleVdByteArray + 1, simpleVdByteArray + 2, simpleVdByteArray + 3, simpleVdByteArray + 4: vt = valueTypeBytes v, _ = d.decodeBytes(nil) - case simpleVdExt: + case simpleVdExt, simpleVdExt + 1, simpleVdExt + 2, simpleVdExt + 3, simpleVdExt + 4: vt = valueTypeExt l := d.decLen() var re RawExt @@ -402,10 +401,10 @@ func (d *simpleDecDriver) decodeNaked() (v interface{}, vt valueType, decodeFurt re.Data = d.r.readn(l) v = &re vt = valueTypeExt - case simpleVdArray: + case simpleVdArray, simpleVdArray + 1, simpleVdArray + 2, simpleVdArray + 3, simpleVdArray + 4: vt = valueTypeArray decodeFurther = true - case simpleVdMap: + case simpleVdMap, simpleVdMap + 1, simpleVdMap + 2, simpleVdMap + 3, simpleVdMap + 4: vt = valueTypeMap decodeFurther = true default: @@ -420,23 +419,34 @@ func (d *simpleDecDriver) decodeNaked() (v interface{}, vt valueType, decodeFurt //------------------------------------ -//SimpleHandle is a Handle for a very simple encoding format. +// SimpleHandle is a Handle for a very simple encoding format. +// +// simple is a simplistic codec similar to binc, but not as compact. +// - Encoding of a value is always preceeded by the descriptor byte (bd) +// - True, false, nil are encoded fully in 1 byte (the descriptor) +// - Integers (intXXX, uintXXX) are encoded in 1, 2, 4 or 8 bytes (plus a descriptor byte). +// There are positive (uintXXX and intXXX >= 0) and negative (intXXX < 0) integers. +// - Floats are encoded in 4 or 8 bytes (plus a descriptor byte) +// - Lenght of containers (strings, bytes, array, map, extensions) +// are encoded in 0, 1, 2, 4 or 8 bytes. +// Zero-length containers have no length encoded. +// For others, the number of bytes is given by pow(2, bd%3) +// - maps are encoded as [bd] [length] [[key][value]]... +// - arrays are encoded as [bd] [length] [value]... +// - extensions are encoded as [bd] [length] [tag] [byte]... +// - strings/bytearrays are encoded as [bd] [length] [byte]... // -//The simple format is similar to binc: -// - All numbers are represented with 8 bytes (int64, uint64, float64) -// - Strings, []byte, arrays and maps have the length pre-pended as a uint64 -// - Thus, there isn't much packing, but the format is extremely simple. -// and easy to create different implementations of (e.g. in C). +// The full spec will be published soon. type SimpleHandle struct { BasicHandle } func (h *SimpleHandle) newEncDriver(w encWriter) encDriver { - return &simpleEncDriver{w: w} + return &simpleEncDriver{w: w, h: h} } func (h *SimpleHandle) newDecDriver(r decReader) decDriver { - return &simpleDecDriver{r: r} + return &simpleDecDriver{r: r, h: h} } func (_ *SimpleHandle) writeExt() bool {