diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..e04276f9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.idea +venv diff --git a/README.md b/README.md index f45b58b1..26f171ee 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,6 @@ -[![Sourcegraph](https://sourcegraph.com/github.com/ugorji/go/-/badge.svg?v=4)](https://sourcegraph.com/github.com/ugorji/go/-/tree/codec?badge) -[![Build Status](https://travis-ci.org/ugorji/go.svg?branch=master)](https://travis-ci.org/ugorji/go) -[![codecov](https://codecov.io/gh/ugorji/go/branch/master/graph/badge.svg?v=4)](https://codecov.io/gh/ugorji/go) -[![GoDoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](http://godoc.org/github.com/ugorji/go/codec) -[![rcard](https://goreportcard.com/badge/github.com/ugorji/go/codec?v=4)](https://goreportcard.com/report/github.com/ugorji/go/codec) -[![License](http://img.shields.io/badge/license-mit-blue.svg?style=flat-square)](https://raw.githubusercontent.com/ugorji/go/master/LICENSE) +# go-msgpack -# go-codec - -This repository contains the `go-codec` library. +This repository contains the `go-msgpack` library. To install: @@ -18,23 +11,13 @@ go get github.com/hashicorp/go-msgpack/codec # Package Documentation -Package codec provides a High Performance, Feature-Rich Idiomatic Go 1.4+ -codec/encoding library for binc, msgpack, cbor, json. +Package codec provides a High Performance, Feature-Rich Idiomatic +codec/encoding library for msgpack, json. Supported Serialization formats are: - msgpack: https://github.com/msgpack/msgpack - - binc: http://github.com/ugorji/binc - - cbor: http://cbor.io http://tools.ietf.org/html/rfc7049 - json: http://json.org http://tools.ietf.org/html/rfc7159 - - simple: - -This package will carefully use 'package unsafe' for performance reasons in -specific places. You can build without unsafe use by passing the safe or -appengine tag i.e. 'go install -tags=safe ...'. Note that unsafe is only -supported for the last 4 go releases e.g. current go release is go 1.12, so -we support unsafe use only from go 1.9+ . This is because supporting unsafe -requires knowledge of implementation details. For detailed usage information, read the primer at http://ugorji.net/blog/go-codec-primer . @@ -45,12 +28,9 @@ standard library (ie json, xml, gob, etc). Rich Feature Set includes: - Simple but extremely powerful and feature-rich API - - Support for go1.4 and above, while selectively using newer APIs for later releases - Excellent code coverage ( > 90% ) - Very High Performance. Our extensive benchmarks show us outperforming Gob, Json, Bson, etc by 2-4X. - - Careful selected use of 'unsafe' for targeted performance gains. - 100% mode exists where 'unsafe' is not used at all. - Lock-free (sans mutex) concurrency for scaling to 100's of cores - In-place updates during decode, with option to zero value in maps and slices prior to decode - Coerce types where appropriate @@ -76,9 +56,9 @@ Rich Feature Set includes: - Comprehensive support for anonymous fields - Fast (no-reflection) encoding/decoding of common maps and slices - Code-generation for faster performance. - - Support binary (e.g. messagepack, cbor) and text (e.g. json) formats + - Support binary (e.g. messagepack) and text (e.g. json) formats - Support indefinite-length formats to enable true streaming - (for formats which support it e.g. json, cbor) + (for formats which support it e.g. json) - Support canonical encoding, where a value is ALWAYS encoded as same sequence of bytes. This mostly applies to maps, where iteration order is non-deterministic. - NIL in data stream decoded as zero value @@ -166,23 +146,21 @@ Sample usage model: ```go // create and configure Handle var ( - bh codec.BincHandle mh codec.MsgpackHandle - ch codec.CborHandle ) mh.MapType = reflect.TypeOf(map[string]interface{}(nil)) // configure extensions // e.g. for msgpack, define functions and enable Time support for tag 1 - // mh.SetExt(reflect.TypeOf(time.Time{}), 1, myExt) + mh.SetExt(reflect.TypeOf(time.Time{}), 1, myExt) // create and use decoder/encoder var ( r io.Reader w io.Writer b []byte - h = &bh // or mh to use msgpack + h = &mh ) dec = codec.NewDecoder(r, h) @@ -235,11 +213,14 @@ You can run the tag 'safe' to run tests or build in safe mode. e.g. ## Running Benchmarks ``` - cd bench + cd codec/bench + ./bench.sh -d + ./bench.sh -c + ./bench.sh -s go test -bench . -benchmem -benchtime 1s ``` -Please see http://github.com/ugorji/go-codec-bench . +Please see http://github.com/hashicorp/go-codec-bench . ## Caveats @@ -251,48 +232,9 @@ decoding - func, complex numbers, unsafe pointers - unexported and not embedded - unexported and embedded and not struct kind - - unexported and embedded pointers (from go1.10) + - unexported and embedded pointers Every other field in a struct will be encoded/decoded. Embedded fields are encoded as if they exist in the top-level struct, with some caveats. See Encode documentation. - -## Exported Package API - -```go -const CborStreamBytes byte = 0x5f ... -const GenVersion = 10 -var GoRpc goRpc -var MsgpackSpecRpc msgpackSpecRpc -func GenHelperDecoder(d *Decoder) (gd genHelperDecoder, dd genHelperDecDriver) -func GenHelperEncoder(e *Encoder) (ge genHelperEncoder, ee genHelperEncDriver) -type BasicHandle struct{ ... } -type BincHandle struct{ ... } -type BytesExt interface{ ... } -type CborHandle struct{ ... } -type DecodeOptions struct{ ... } -type Decoder struct{ ... } - func NewDecoder(r io.Reader, h Handle) *Decoder - func NewDecoderBytes(in []byte, h Handle) *Decoder -type EncodeOptions struct{ ... } -type Encoder struct{ ... } - func NewEncoder(w io.Writer, h Handle) *Encoder - func NewEncoderBytes(out *[]byte, h Handle) *Encoder -type Ext interface{ ... } -type Handle interface{ ... } -type InterfaceExt interface{ ... } -type JsonHandle struct{ ... } -type MapBySlice interface{ ... } -type MissingFielder interface{ ... } -type MsgpackHandle struct{ ... } -type MsgpackSpecRpcMultiArgs []interface{} -type RPCOptions struct{ ... } -type Raw []byte -type RawExt struct{ ... } -type Rpc interface{ ... } -type Selfer interface{ ... } -type SimpleHandle struct{ ... } -type TypeInfos struct{ ... } - func NewTypeInfos(tags []string) *TypeInfos -``` diff --git a/codec/bench/README.md b/codec/bench/README.md index 170719d2..38a04de3 100644 --- a/codec/bench/README.md +++ b/codec/bench/README.md @@ -2,15 +2,12 @@ This is a comparison of different binary and text encodings. -We compare the codecs provided by github.com/ugorji/go/codec package, +We compare the codecs provided by github.com/hashicorp/go-msgpack/v2/codec package, against other libraries: -[github.com/ugorji/go/codec](http://github.com/ugorji/go) provides: +[github.com/go-msgpack/go-msgpack/v2/codec](http://github.com/hashicorp/go-msgpack/v2/codec) provides: - msgpack: [http://github.com/msgpack/msgpack] - - binc: [http://github.com/ugorji/binc] - - cbor: [http://cbor.io] [http://tools.ietf.org/html/rfc7049] - - simple: - json: [http://json.org] [http://tools.ietf.org/html/rfc7159] Other codecs compared include: @@ -19,11 +16,9 @@ Other codecs compared include: - [gopkg.in/mgo.v2/bson](http://gopkg.in/mgo.v2/bson) - [github.com/davecgh/go-xdr/xdr2](https://godoc.org/github.com/davecgh/go-xdr/xdr) - [github.com/Sereal/Sereal/Go/sereal](https://godoc.org/github.com/Sereal/Sereal/Go/sereal) - - [code.google.com/p/cbor/go](http://code.google.com/p/cbor/go) - [github.com/tinylib/msgp](http://github.com/tinylib/msgp) - [github.com/tinylib/msgp](http://godoc.org/github.com/tinylib/msgp) - [github.com/pquerna/ffjson/ffjson](http://godoc.org/github.com/pquerna/ffjson/ffjson) - - [bitbucket.org/bodhisnarkva/cbor/go](http://godoc.org/bitbucket.org/bodhisnarkva/cbor/go) - [github.com/json-iterator/go](http://godoc.org/github.com/json-iterator/go) - [github.com/mailru/easyjson](http://godoc.org/github.com/mailru/easyjson) diff --git a/codec/bench/bench.sh b/codec/bench/bench.sh index a8aaf101..e6b2070e 100755 --- a/codec/bench/bench.sh +++ b/codec/bench/bench.sh @@ -1,18 +1,24 @@ #!/bin/bash -# download the code and all its dependencies +# download the code and all its dependencies _go_get() { go get -u \ - "github.com/ugorji/go/codec" "github.com/ugorji/go/codec"/codecgen \ - github.com/tinylib/msgp/msgp github.com/tinylib/msgp \ - github.com/pquerna/ffjson/ffjson github.com/pquerna/ffjson \ + github.com/hashicorp/go-msgpack/v2/codec \ + github.com/hashicorp/go-msgpack/v2/codec/codecgen \ + github.com/tinylib/msgp/msgp \ + github.com/tinylib/msgp \ + github.com/pquerna/ffjson/ffjson \ + github.com/pquerna/ffjson \ github.com/Sereal/Sereal/Go/sereal \ - bitbucket.org/bodhisnarkva/cbor/go \ github.com/davecgh/go-xdr/xdr2 \ gopkg.in/mgo.v2/bson \ gopkg.in/vmihailenco/msgpack.v2 \ github.com/json-iterator/go \ github.com/mailru/easyjson/... + go install github.com/tinylib/msgp@latest + go get github.com/mailru/easyjson && go install github.com/mailru/easyjson/...@latest + go install github.com/pquerna/ffjson@latest + } # add generated tag to the top of each file @@ -45,7 +51,7 @@ _gen() { echo "easyjson ... " && easyjson -all -no_std_marshalers -omit_empty -output_filename e9.go v.go && _prependbt e9.go values_easyjson${zsfx} && - echo "ffjson ... " && + echo "ffjson ... " && ffjson -force-regenerate -reset-fields -w f9.go v.go && _prependbt f9.go values_ffjson${zsfx} && sed -i '' -e 's+ MarshalJSON(+ _MarshalJSON(+g' values_ffjson${zsfx} && @@ -59,11 +65,11 @@ _gen() { # # Basically, its a sequence of # go test -tags "alltests x safe codecgen generated" -bench "CodecSuite or AllSuite or XSuite" -benchmem -# +# _suite() { local t="alltests x" - local a=( "" "safe" "codecgen" "codecgen safe") - local b=( "generated" "generated safe") + local a=( "" "codecgen" ) + local b=( "generated" ) for i in "${a[@]}" do echo ">>>> bench TAGS: '$t $i' SUITE: BenchmarkCodecXSuite" @@ -104,4 +110,4 @@ then else echo "bench.sh must be run from the directory it resides in" _usage -fi +fi diff --git a/codec/bench/bench_test.go b/codec/bench/bench_test.go index 6e108b7b..35e5227f 100644 --- a/codec/bench/bench_test.go +++ b/codec/bench/bench_test.go @@ -12,6 +12,8 @@ import ( "runtime" "testing" "time" + + "github.com/hashicorp/go-msgpack/v2/codec/internal" ) // Sample way to run: @@ -46,7 +48,7 @@ func benchReinit() { func benchPreInit() { benchTs = newTestStruc(benchDepth, testNumRepeatString, true, !testSkipIntf, benchMapStringKeyOnly) - approxSize = approxDataSize(reflect.ValueOf(benchTs)) * 3 / 2 // multiply by 1.5 to appease msgp, and prevent alloc + approxSize = internal.ApproxDataSize(reflect.ValueOf(benchTs)) * 3 / 2 // multiply by 1.5 to appease msgp, and prevent alloc // bytesLen := 1024 * 4 * (benchDepth + 1) * (benchDepth + 1) // if bytesLen < approxSize { // bytesLen = approxSize @@ -54,9 +56,6 @@ func benchPreInit() { benchCheckers = append(benchCheckers, benchChecker{"msgpack", fnMsgpackEncodeFn, fnMsgpackDecodeFn}, - benchChecker{"binc", fnBincEncodeFn, fnBincDecodeFn}, - benchChecker{"simple", fnSimpleEncodeFn, fnSimpleDecodeFn}, - benchChecker{"cbor", fnCborEncodeFn, fnCborDecodeFn}, benchChecker{"json", fnJsonEncodeFn, fnJsonDecodeFn}, benchChecker{"std-json", fnStdJsonEncodeFn, fnStdJsonDecodeFn}, benchChecker{"gob", fnGobEncodeFn, fnGobDecodeFn}, @@ -139,7 +138,7 @@ func doBenchCheck(name string, encfn benchEncFn, decfn benchDecFn) { decDur := time.Since(tnow) // if benchCheckDoDeepEqual { if benchVerify { - err = deepEqual(benchTs, &ts2) + err = internal.DeepEqual(benchTs, &ts2) if err == nil { logT(nil, "\t%10s: len: %d bytes,\t encode: %v,\t decode: %v,\tencoded = decoded", name, encLen, encDur, decDur) } else { @@ -205,19 +204,6 @@ func fnBenchmarkDecode(b *testing.B, encName string, ts interface{}, logT(b, "Error encoding benchTs: %s: %v", encName, err) b.FailNow() } - if false && benchVerify { // do not do benchVerify during decode - // ts2 := newfn() - ts1 := ts.(*TestStruc) - ts2 := new(TestStruc) - if err = decfn(buf, ts2); err != nil { - logT(b, "BenchVerify: Error decoding benchTs: %s: %v", encName, err) - b.FailNow() - } - if err = deepEqual(ts1, ts2); err != nil { - logT(b, "BenchVerify: Error comparing benchTs: %s: %v", encName, err) - b.FailNow() - } - } runtime.GC() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -242,30 +228,6 @@ func fnMsgpackDecodeFn(buf []byte, ts interface{}) error { return sTestCodecDecode(buf, ts, testMsgpackH, &testMsgpackH.BasicHandle) } -func fnBincEncodeFn(ts interface{}, bsIn []byte) (bs []byte, err error) { - return sTestCodecEncode(ts, bsIn, fnBenchmarkByteBuf, testBincH, &testBincH.BasicHandle) -} - -func fnBincDecodeFn(buf []byte, ts interface{}) error { - return sTestCodecDecode(buf, ts, testBincH, &testBincH.BasicHandle) -} - -func fnSimpleEncodeFn(ts interface{}, bsIn []byte) (bs []byte, err error) { - return sTestCodecEncode(ts, bsIn, fnBenchmarkByteBuf, testSimpleH, &testSimpleH.BasicHandle) -} - -func fnSimpleDecodeFn(buf []byte, ts interface{}) error { - return sTestCodecDecode(buf, ts, testSimpleH, &testSimpleH.BasicHandle) -} - -func fnCborEncodeFn(ts interface{}, bsIn []byte) (bs []byte, err error) { - return sTestCodecEncode(ts, bsIn, fnBenchmarkByteBuf, testCborH, &testCborH.BasicHandle) -} - -func fnCborDecodeFn(buf []byte, ts interface{}) error { - return sTestCodecDecode(buf, ts, testCborH, &testCborH.BasicHandle) -} - func fnJsonEncodeFn(ts interface{}, bsIn []byte) (bs []byte, err error) { return sTestCodecEncode(ts, bsIn, fnBenchmarkByteBuf, testJsonH, &testJsonH.BasicHandle) } @@ -316,18 +278,6 @@ func Benchmark__Msgpack____Encode(b *testing.B) { fnBenchmarkEncode(b, "msgpack", benchTs, fnMsgpackEncodeFn) } -func Benchmark__Binc_______Encode(b *testing.B) { - fnBenchmarkEncode(b, "binc", benchTs, fnBincEncodeFn) -} - -func Benchmark__Simple_____Encode(b *testing.B) { - fnBenchmarkEncode(b, "simple", benchTs, fnSimpleEncodeFn) -} - -func Benchmark__Cbor_______Encode(b *testing.B) { - fnBenchmarkEncode(b, "cbor", benchTs, fnCborEncodeFn) -} - func Benchmark__Json_______Encode(b *testing.B) { fnBenchmarkEncode(b, "json", benchTs, fnJsonEncodeFn) } @@ -350,18 +300,6 @@ func Benchmark__Msgpack____Decode(b *testing.B) { fnBenchmarkDecode(b, "msgpack", benchTs, fnMsgpackEncodeFn, fnMsgpackDecodeFn, fnBenchNewTs) } -func Benchmark__Binc_______Decode(b *testing.B) { - fnBenchmarkDecode(b, "binc", benchTs, fnBincEncodeFn, fnBincDecodeFn, fnBenchNewTs) -} - -func Benchmark__Simple_____Decode(b *testing.B) { - fnBenchmarkDecode(b, "simple", benchTs, fnSimpleEncodeFn, fnSimpleDecodeFn, fnBenchNewTs) -} - -func Benchmark__Cbor_______Decode(b *testing.B) { - fnBenchmarkDecode(b, "cbor", benchTs, fnCborEncodeFn, fnCborDecodeFn, fnBenchNewTs) -} - func Benchmark__Json_______Decode(b *testing.B) { fnBenchmarkDecode(b, "json", benchTs, fnJsonEncodeFn, fnJsonDecodeFn, fnBenchNewTs) } diff --git a/codec/bench/shared_test.go b/codec/bench/shared_test.go index f2503bdc..c9df812c 100644 --- a/codec/bench/shared_test.go +++ b/codec/bench/shared_test.go @@ -49,9 +49,9 @@ import ( "log" "sync" "testing" -) -import . "github.com/hashicorp/go-msgpack/v2/codec" + . "github.com/hashicorp/go-msgpack/v2/codec" +) type testHED struct { H Handle @@ -78,9 +78,6 @@ func (x ioWriterWrapper) Write(p []byte) (n int, err error) { var ( // testNoopH = NoopHandle(8) testMsgpackH = &MsgpackHandle{} - testBincH = &BincHandle{} - testSimpleH = &SimpleHandle{} - testCborH = &CborHandle{} testJsonH = &JsonHandle{} testHandles []Handle @@ -139,12 +136,9 @@ func init() { testHEDs = make([]testHED, 0, 32) testHandles = append(testHandles, // testNoopH, - testMsgpackH, testBincH, testSimpleH, testCborH, testJsonH) + testMsgpackH, testJsonH) // set ExplicitRelease on each handle testMsgpackH.ExplicitRelease = true - testBincH.ExplicitRelease = true - testSimpleH.ExplicitRelease = true - testCborH.ExplicitRelease = true testJsonH.ExplicitRelease = true testInitFlags() @@ -286,8 +280,10 @@ func sTestCodecDecode(bs []byte, ts interface{}, h Handle, bh *BasicHandle) (err func logT(x interface{}, format string, args ...interface{}) { if t, ok := x.(*testing.T); ok && t != nil { + t.Helper() t.Logf(format, args...) } else if b, ok := x.(*testing.B); ok && b != nil { + b.Helper() b.Logf(format, args...) } else { // if testing.Verbose() { // if testVerbose { if len(format) == 0 || format[len(format)-1] != '\n' { diff --git a/codec/bench/x_bench_gen_test.go b/codec/bench/x_bench_gen_test.go index 0378e10e..ed9aeb95 100644 --- a/codec/bench/x_bench_gen_test.go +++ b/codec/bench/x_bench_gen_test.go @@ -1,5 +1,4 @@ //go:build x && generated -// +build x,generated // Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. // Use of this source code is governed by a MIT license found in the LICENSE file. diff --git a/codec/bench/x_bench_test.go b/codec/bench/x_bench_test.go index ce812129..35bf53c3 100644 --- a/codec/bench/x_bench_test.go +++ b/codec/bench/x_bench_test.go @@ -1,5 +1,4 @@ //go:build x -// +build x // Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. // Use of this source code is governed by a MIT license found in the LICENSE file. @@ -10,7 +9,6 @@ import ( "bytes" "testing" - gcbor "bitbucket.org/bodhisnarkva/cbor/go" // gcbor "code.google.com/p/cbor/go" "github.com/Sereal/Sereal/Go/sereal" "github.com/davecgh/go-xdr/xdr2" "github.com/json-iterator/go" @@ -23,7 +21,6 @@ import ( go get -u github.com/tinylib/msgp/msgp github.com/tinylib/msgp \ github.com/pquerna/ffjson/ffjson github.com/pquerna/ffjson \ github.com/Sereal/Sereal/Go/sereal \ - bitbucket.org/bodhisnarkva/cbor/go \ github.com/davecgh/go-xdr/xdr2 \ gopkg.in/mgo.v2/bson \ gopkg.in/vmihailenco/msgpack.v2 \ @@ -45,7 +42,6 @@ func benchXPreInit() { benchChecker{"v-msgpack", fnVMsgpackEncodeFn, fnVMsgpackDecodeFn}, benchChecker{"bson", fnBsonEncodeFn, fnBsonDecodeFn}, // place codecs with issues at the end, so as not to make results too ugly - benchChecker{"gcbor", fnGcborEncodeFn, fnGcborDecodeFn}, // this logs fat ugly message, but we log.SetOutput(ioutil.Discard) benchChecker{"xdr", fnXdrEncodeFn, fnXdrDecodeFn}, benchChecker{"sereal", fnSerealEncodeFn, fnSerealDecodeFn}, ) @@ -110,16 +106,6 @@ func fnSerealDecodeFn(buf []byte, ts interface{}) error { return sereal.Unmarshal(buf, ts) } -func fnGcborEncodeFn(ts interface{}, bsIn []byte) (bs []byte, err error) { - buf := fnBenchmarkByteBuf(bsIn) - err = gcbor.NewEncoder(buf).Encode(ts) - return buf.Bytes(), err -} - -func fnGcborDecodeFn(buf []byte, ts interface{}) error { - return gcbor.NewDecoder(bytes.NewReader(buf)).Decode(ts) -} - func Benchmark__JsonIter___Encode(b *testing.B) { fnBenchmarkEncode(b, "jsoniter", benchTs, fnJsonIterEncodeFn) } @@ -146,14 +132,6 @@ func Benchmark__VMsgpack___Decode(b *testing.B) { fnBenchmarkDecode(b, "v-msgpack", benchTs, fnVMsgpackEncodeFn, fnVMsgpackDecodeFn, fnBenchNewTs) } -func Benchmark__Gcbor______Encode(b *testing.B) { - fnBenchmarkEncode(b, "gcbor", benchTs, fnGcborEncodeFn) -} - -func Benchmark__Gcbor______Decode(b *testing.B) { - fnBenchmarkDecode(b, "gcbor", benchTs, fnGcborEncodeFn, fnGcborDecodeFn, fnBenchNewTs) -} - func Benchmark__Xdr________Encode(b *testing.B) { fnBenchmarkEncode(b, "xdr", benchTs, fnXdrEncodeFn) } diff --git a/codec/bench/z_all_bench_test.go b/codec/bench/z_all_bench_test.go index 440619df..05ae5a47 100644 --- a/codec/bench/z_all_bench_test.go +++ b/codec/bench/z_all_bench_test.go @@ -1,9 +1,7 @@ // Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. // Use of this source code is governed by a MIT license found in the LICENSE file. -//go:build (alltests || codecgen) && go1.7 -// +build alltests codecgen -// +build go1.7 +//go:build alltests || codecgen package codec @@ -14,9 +12,9 @@ import ( "sync" "testing" "time" -) -import . "github.com/ugorji/go/codec" + . "github.com/hashicorp/go-msgpack/v2/codec" +) var benchmarkGroupOnce sync.Once @@ -196,18 +194,12 @@ find . -name "$z" | xargs grep -e '^func Benchmark.*Decode' | \ func benchmarkCodecGroup(t *testing.B) { benchmarkDivider() t.Run("Benchmark__Msgpack____Encode", Benchmark__Msgpack____Encode) - t.Run("Benchmark__Binc_______Encode", Benchmark__Binc_______Encode) - t.Run("Benchmark__Simple_____Encode", Benchmark__Simple_____Encode) - t.Run("Benchmark__Cbor_______Encode", Benchmark__Cbor_______Encode) t.Run("Benchmark__Json_______Encode", Benchmark__Json_______Encode) t.Run("Benchmark__Std_Json___Encode", Benchmark__Std_Json___Encode) t.Run("Benchmark__Gob________Encode", Benchmark__Gob________Encode) // t.Run("Benchmark__Std_Xml____Encode", Benchmark__Std_Xml____Encode) benchmarkDivider() t.Run("Benchmark__Msgpack____Decode", Benchmark__Msgpack____Decode) - t.Run("Benchmark__Binc_______Decode", Benchmark__Binc_______Decode) - t.Run("Benchmark__Simple_____Decode", Benchmark__Simple_____Decode) - t.Run("Benchmark__Cbor_______Decode", Benchmark__Cbor_______Decode) t.Run("Benchmark__Json_______Decode", Benchmark__Json_______Decode) t.Run("Benchmark__Std_Json___Decode", Benchmark__Std_Json___Decode) t.Run("Benchmark__Gob________Decode", Benchmark__Gob________Decode) @@ -224,17 +216,7 @@ func benchmarkJsonDecodeGroup(t *testing.B) { t.Run("Benchmark__Json_______Decode", Benchmark__Json_______Decode) } -func benchmarkCborEncodeGroup(t *testing.B) { - t.Run("Benchmark__Cbor_______Encode", Benchmark__Cbor_______Encode) -} - -func benchmarkCborDecodeGroup(t *testing.B) { - t.Run("Benchmark__Cbor_______Decode", Benchmark__Cbor_______Decode) -} - func BenchmarkCodecQuickSuite(t *testing.B) { - benchmarkQuickSuite(t, "cbor", benchmarkCborEncodeGroup) - benchmarkQuickSuite(t, "cbor", benchmarkCborDecodeGroup) benchmarkQuickSuite(t, "json", benchmarkJsonEncodeGroup) benchmarkQuickSuite(t, "json", benchmarkJsonDecodeGroup) diff --git a/codec/bench/z_all_x_bench_gen_test.go b/codec/bench/z_all_x_bench_gen_test.go index f9bbb426..7bad4c89 100644 --- a/codec/bench/z_all_x_bench_gen_test.go +++ b/codec/bench/z_all_x_bench_gen_test.go @@ -1,8 +1,7 @@ // Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. // Use of this source code is governed by a MIT license found in the LICENSE file. -//go:build alltests && x && go1.7 && generated -// +build alltests,x,go1.7,generated +//go:build alltests && x && generated package codec @@ -13,9 +12,6 @@ import "testing" func benchmarkCodecXGenGroup(t *testing.B) { benchmarkDivider() t.Run("Benchmark__Msgpack____Encode", Benchmark__Msgpack____Encode) - t.Run("Benchmark__Binc_______Encode", Benchmark__Binc_______Encode) - t.Run("Benchmark__Simple_____Encode", Benchmark__Simple_____Encode) - t.Run("Benchmark__Cbor_______Encode", Benchmark__Cbor_______Encode) t.Run("Benchmark__Json_______Encode", Benchmark__Json_______Encode) t.Run("Benchmark__Std_Json___Encode", Benchmark__Std_Json___Encode) t.Run("Benchmark__Gob________Encode", Benchmark__Gob________Encode) @@ -31,16 +27,13 @@ func benchmarkCodecXGenGroup(t *testing.B) { benchmarkDivider() t.Run("Benchmark__Msgpack____Decode", Benchmark__Msgpack____Decode) - t.Run("Benchmark__Binc_______Decode", Benchmark__Binc_______Decode) - t.Run("Benchmark__Simple_____Decode", Benchmark__Simple_____Decode) - t.Run("Benchmark__Cbor_______Decode", Benchmark__Cbor_______Decode) t.Run("Benchmark__Json_______Decode", Benchmark__Json_______Decode) t.Run("Benchmark__Std_Json___Decode", Benchmark__Std_Json___Decode) t.Run("Benchmark__Gob________Decode", Benchmark__Gob________Decode) t.Run("Benchmark__JsonIter___Decode", Benchmark__JsonIter___Decode) t.Run("Benchmark__Bson_______Decode", Benchmark__Bson_______Decode) // t.Run("Benchmark__VMsgpack___Decode", Benchmark__VMsgpack___Decode) - t.Run("Benchmark__Msgp_______Decode", Benchmark__Msgp_______Decode) + //t.Run("Benchmark__Msgp_______Decode", Benchmark__Msgp_______Decode) // fails t.Run("Benchmark__Easyjson___Decode", Benchmark__Easyjson___Decode) t.Run("Benchmark__Ffjson_____Decode", Benchmark__Ffjson_____Decode) // t.Run("Benchmark__Gcbor______Decode", Benchmark__Gcbor______Decode) diff --git a/codec/bench/z_all_x_bench_test.go b/codec/bench/z_all_x_bench_test.go index 2ea84896..3f6e3315 100644 --- a/codec/bench/z_all_x_bench_test.go +++ b/codec/bench/z_all_x_bench_test.go @@ -1,8 +1,7 @@ // Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. // Use of this source code is governed by a MIT license found in the LICENSE file. -//go:build alltests && x && go1.7 -// +build alltests,x,go1.7 +//go:build alltests && x package codec @@ -38,9 +37,6 @@ func benchmarkXGroup(t *testing.B) { func benchmarkCodecXGroup(t *testing.B) { benchmarkDivider() t.Run("Benchmark__Msgpack____Encode", Benchmark__Msgpack____Encode) - t.Run("Benchmark__Binc_______Encode", Benchmark__Binc_______Encode) - t.Run("Benchmark__Simple_____Encode", Benchmark__Simple_____Encode) - t.Run("Benchmark__Cbor_______Encode", Benchmark__Cbor_______Encode) t.Run("Benchmark__Json_______Encode", Benchmark__Json_______Encode) t.Run("Benchmark__Std_Json___Encode", Benchmark__Std_Json___Encode) t.Run("Benchmark__Gob________Encode", Benchmark__Gob________Encode) @@ -54,9 +50,6 @@ func benchmarkCodecXGroup(t *testing.B) { benchmarkDivider() t.Run("Benchmark__Msgpack____Decode", Benchmark__Msgpack____Decode) - t.Run("Benchmark__Binc_______Decode", Benchmark__Binc_______Decode) - t.Run("Benchmark__Simple_____Decode", Benchmark__Simple_____Decode) - t.Run("Benchmark__Cbor_______Decode", Benchmark__Cbor_______Decode) t.Run("Benchmark__Json_______Decode", Benchmark__Json_______Decode) t.Run("Benchmark__Std_Json___Decode", Benchmark__Std_Json___Decode) t.Run("Benchmark__Gob________Decode", Benchmark__Gob________Decode) diff --git a/codec/binc.go b/codec/binc.go deleted file mode 100644 index 03fbb535..00000000 --- a/codec/binc.go +++ /dev/null @@ -1,1202 +0,0 @@ -// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -package codec - -import ( - "math" - "reflect" - "time" -) - -const bincDoPrune = true // No longer needed. Needed before as C lib did not support pruning. - -// vd as low 4 bits (there are 16 slots) -const ( - bincVdSpecial byte = iota - bincVdPosInt - bincVdNegInt - bincVdFloat - - bincVdString - bincVdByteArray - bincVdArray - bincVdMap - - bincVdTimestamp - bincVdSmallInt - bincVdUnicodeOther - bincVdSymbol - - bincVdDecimal - _ // open slot - _ // open slot - bincVdCustomExt = 0x0f -) - -const ( - bincSpNil byte = iota - bincSpFalse - bincSpTrue - bincSpNan - bincSpPosInf - bincSpNegInf - bincSpZeroFloat - bincSpZero - bincSpNegOne -) - -const ( - bincFlBin16 byte = iota - bincFlBin32 - _ // bincFlBin32e - bincFlBin64 - _ // bincFlBin64e - // others not currently supported -) - -func bincdesc(vd, vs byte) string { - switch vd { - case bincVdSpecial: - switch vs { - case bincSpNil: - return "nil" - case bincSpFalse: - return "false" - case bincSpTrue: - return "true" - case bincSpNan, bincSpPosInf, bincSpNegInf, bincSpZeroFloat: - return "float" - case bincSpZero: - return "uint" - case bincSpNegOne: - return "int" - default: - return "unknown" - } - case bincVdSmallInt, bincVdPosInt: - return "uint" - case bincVdNegInt: - return "int" - case bincVdFloat: - return "float" - case bincVdSymbol: - return "string" - case bincVdString: - return "string" - case bincVdByteArray: - return "bytes" - case bincVdTimestamp: - return "time" - case bincVdCustomExt: - return "ext" - case bincVdArray: - return "array" - case bincVdMap: - return "map" - default: - return "unknown" - } -} - -type bincEncDriver struct { - e *Encoder - h *BincHandle - w *encWriterSwitch - m map[string]uint16 // symbols - b [16]byte // scratch, used for encoding numbers - bigendian style - s uint16 // symbols sequencer - // c containerState - encDriverTrackContainerWriter - noBuiltInTypes - // encNoSeparator - _ [1]uint64 // padding -} - -func (e *bincEncDriver) EncodeNil() { - e.w.writen1(bincVdSpecial<<4 | bincSpNil) -} - -func (e *bincEncDriver) EncodeTime(t time.Time) { - if t.IsZero() { - e.EncodeNil() - } else { - bs := bincEncodeTime(t) - e.w.writen1(bincVdTimestamp<<4 | uint8(len(bs))) - e.w.writeb(bs) - } -} - -func (e *bincEncDriver) EncodeBool(b bool) { - if b { - e.w.writen1(bincVdSpecial<<4 | bincSpTrue) - } else { - e.w.writen1(bincVdSpecial<<4 | bincSpFalse) - } -} - -func (e *bincEncDriver) EncodeFloat32(f float32) { - if f == 0 { - e.w.writen1(bincVdSpecial<<4 | bincSpZeroFloat) - return - } - e.w.writen1(bincVdFloat<<4 | bincFlBin32) - bigenHelper{e.b[:4], e.w}.writeUint32(math.Float32bits(f)) -} - -func (e *bincEncDriver) EncodeFloat64(f float64) { - if f == 0 { - e.w.writen1(bincVdSpecial<<4 | bincSpZeroFloat) - return - } - bigen.PutUint64(e.b[:8], math.Float64bits(f)) - if bincDoPrune { - i := 7 - for ; i >= 0 && (e.b[i] == 0); i-- { - } - i++ - if i <= 6 { - e.w.writen1(bincVdFloat<<4 | 0x8 | bincFlBin64) - e.w.writen1(byte(i)) - e.w.writeb(e.b[:i]) - return - } - } - e.w.writen1(bincVdFloat<<4 | bincFlBin64) - e.w.writeb(e.b[:8]) -} - -func (e *bincEncDriver) encIntegerPrune(bd byte, pos bool, v uint64, lim uint8) { - if lim == 4 { - bigen.PutUint32(e.b[:lim], uint32(v)) - } else { - bigen.PutUint64(e.b[:lim], v) - } - if bincDoPrune { - i := pruneSignExt(e.b[:lim], pos) - e.w.writen1(bd | lim - 1 - byte(i)) - e.w.writeb(e.b[i:lim]) - } else { - e.w.writen1(bd | lim - 1) - e.w.writeb(e.b[:lim]) - } -} - -func (e *bincEncDriver) EncodeInt(v int64) { - // const nbd byte = bincVdNegInt << 4 - if v >= 0 { - e.encUint(bincVdPosInt<<4, true, uint64(v)) - } else if v == -1 { - e.w.writen1(bincVdSpecial<<4 | bincSpNegOne) - } else { - e.encUint(bincVdNegInt<<4, false, uint64(-v)) - } -} - -func (e *bincEncDriver) EncodeUint(v uint64) { - e.encUint(bincVdPosInt<<4, true, v) -} - -func (e *bincEncDriver) encUint(bd byte, pos bool, v uint64) { - if v == 0 { - e.w.writen1(bincVdSpecial<<4 | bincSpZero) - } else if pos && v >= 1 && v <= 16 { - e.w.writen1(bincVdSmallInt<<4 | byte(v-1)) - } else if v <= math.MaxUint8 { - e.w.writen2(bd|0x0, byte(v)) - } else if v <= math.MaxUint16 { - e.w.writen1(bd | 0x01) - bigenHelper{e.b[:2], e.w}.writeUint16(uint16(v)) - } else if v <= math.MaxUint32 { - e.encIntegerPrune(bd, pos, v, 4) - } else { - e.encIntegerPrune(bd, pos, v, 8) - } -} - -func (e *bincEncDriver) EncodeExt(rv interface{}, xtag uint64, ext Ext, _ *Encoder) { - bs := ext.WriteExt(rv) - if bs == nil { - e.EncodeNil() - return - } - e.encodeExtPreamble(uint8(xtag), len(bs)) - e.w.writeb(bs) -} - -func (e *bincEncDriver) EncodeRawExt(re *RawExt, _ *Encoder) { - e.encodeExtPreamble(uint8(re.Tag), len(re.Data)) - e.w.writeb(re.Data) -} - -func (e *bincEncDriver) encodeExtPreamble(xtag byte, length int) { - e.encLen(bincVdCustomExt<<4, uint64(length)) - e.w.writen1(xtag) -} - -func (e *bincEncDriver) WriteArrayStart(length int) { - e.encLen(bincVdArray<<4, uint64(length)) - e.c = containerArrayStart -} - -func (e *bincEncDriver) WriteMapStart(length int) { - e.encLen(bincVdMap<<4, uint64(length)) - e.c = containerMapStart -} - -func (e *bincEncDriver) EncodeSymbol(v string) { - // if WriteSymbolsNoRefs { - // e.encodeString(cUTF8, v) - // return - // } - - //symbols only offer benefit when string length > 1. - //This is because strings with length 1 take only 2 bytes to store - //(bd with embedded length, and single byte for string val). - - l := len(v) - if l == 0 { - e.encBytesLen(cUTF8, 0) - return - } else if l == 1 { - e.encBytesLen(cUTF8, 1) - e.w.writen1(v[0]) - return - } - if e.m == nil { - e.m = make(map[string]uint16, 16) - } - ui, ok := e.m[v] - if ok { - if ui <= math.MaxUint8 { - e.w.writen2(bincVdSymbol<<4, byte(ui)) - } else { - e.w.writen1(bincVdSymbol<<4 | 0x8) - bigenHelper{e.b[:2], e.w}.writeUint16(ui) - } - } else { - e.s++ - ui = e.s - //ui = uint16(atomic.AddUint32(&e.s, 1)) - e.m[v] = ui - var lenprec uint8 - if l <= math.MaxUint8 { - // lenprec = 0 - } else if l <= math.MaxUint16 { - lenprec = 1 - } else if int64(l) <= math.MaxUint32 { - lenprec = 2 - } else { - lenprec = 3 - } - if ui <= math.MaxUint8 { - e.w.writen2(bincVdSymbol<<4|0x0|0x4|lenprec, byte(ui)) - } else { - e.w.writen1(bincVdSymbol<<4 | 0x8 | 0x4 | lenprec) - bigenHelper{e.b[:2], e.w}.writeUint16(ui) - } - if lenprec == 0 { - e.w.writen1(byte(l)) - } else if lenprec == 1 { - bigenHelper{e.b[:2], e.w}.writeUint16(uint16(l)) - } else if lenprec == 2 { - bigenHelper{e.b[:4], e.w}.writeUint32(uint32(l)) - } else { - bigenHelper{e.b[:8], e.w}.writeUint64(uint64(l)) - } - e.w.writestr(v) - } -} - -func (e *bincEncDriver) EncodeString(c charEncoding, v string) { - if e.c == containerMapKey && c == cUTF8 && (e.h.AsSymbols == 0 || e.h.AsSymbols == 1) { - e.EncodeSymbol(v) - return - } - l := uint64(len(v)) - e.encBytesLen(c, l) - if l > 0 { - e.w.writestr(v) - } -} - -func (e *bincEncDriver) EncodeStringEnc(c charEncoding, v string) { - if e.c == containerMapKey && c == cUTF8 && (e.h.AsSymbols == 0 || e.h.AsSymbols == 1) { - e.EncodeSymbol(v) - return - } - l := uint64(len(v)) - e.encLen(bincVdString<<4, l) // e.encBytesLen(c, l) - if l > 0 { - e.w.writestr(v) - } - -} - -func (e *bincEncDriver) EncodeStringBytes(c charEncoding, v []byte) { - if v == nil { - e.EncodeNil() - return - } - l := uint64(len(v)) - e.encBytesLen(c, l) - if l > 0 { - e.w.writeb(v) - } -} - -func (e *bincEncDriver) EncodeStringBytesRaw(v []byte) { - if v == nil { - e.EncodeNil() - return - } - l := uint64(len(v)) - e.encLen(bincVdByteArray<<4, l) // e.encBytesLen(c, l) - if l > 0 { - e.w.writeb(v) - } -} - -func (e *bincEncDriver) encBytesLen(c charEncoding, length uint64) { - //TODO: support bincUnicodeOther (for now, just use string or bytearray) - if c == cRAW { - e.encLen(bincVdByteArray<<4, length) - } else { - e.encLen(bincVdString<<4, length) - } -} - -func (e *bincEncDriver) encLen(bd byte, l uint64) { - if l < 12 { - e.w.writen1(bd | uint8(l+4)) - } else { - e.encLenNumber(bd, l) - } -} - -func (e *bincEncDriver) encLenNumber(bd byte, v uint64) { - if v <= math.MaxUint8 { - e.w.writen2(bd, byte(v)) - } else if v <= math.MaxUint16 { - e.w.writen1(bd | 0x01) - bigenHelper{e.b[:2], e.w}.writeUint16(uint16(v)) - } else if v <= math.MaxUint32 { - e.w.writen1(bd | 0x02) - bigenHelper{e.b[:4], e.w}.writeUint32(uint32(v)) - } else { - e.w.writen1(bd | 0x03) - bigenHelper{e.b[:8], e.w}.writeUint64(uint64(v)) - } -} - -//------------------------------------ - -type bincDecSymbol struct { - s string - b []byte - i uint16 -} - -type bincDecDriver struct { - decDriverNoopContainerReader - noBuiltInTypes - - d *Decoder - h *BincHandle - r *decReaderSwitch - br bool // bytes reader - bdRead bool - bd byte - vd byte - vs byte - _ [3]byte // padding - // linear searching on this slice is ok, - // because we typically expect < 32 symbols in each stream. - s []bincDecSymbol - - // noStreamingCodec - // decNoSeparator - - b [(8 + 1) * 8]byte // scratch -} - -func (d *bincDecDriver) readNextBd() { - d.bd = d.r.readn1() - d.vd = d.bd >> 4 - d.vs = d.bd & 0x0f - d.bdRead = true -} - -func (d *bincDecDriver) uncacheRead() { - if d.bdRead { - d.r.unreadn1() - d.bdRead = false - } -} - -func (d *bincDecDriver) ContainerType() (vt valueType) { - if !d.bdRead { - d.readNextBd() - } - if d.vd == bincVdSpecial && d.vs == bincSpNil { - return valueTypeNil - } else if d.vd == bincVdByteArray { - return valueTypeBytes - } else if d.vd == bincVdString { - return valueTypeString - } else if d.vd == bincVdArray { - return valueTypeArray - } else if d.vd == bincVdMap { - return valueTypeMap - } - // else { - // d.d.errorf("isContainerType: unsupported parameter: %v", vt) - // } - return valueTypeUnset -} - -func (d *bincDecDriver) TryDecodeAsNil() bool { - if !d.bdRead { - d.readNextBd() - } - if d.bd == bincVdSpecial<<4|bincSpNil { - d.bdRead = false - return true - } - return false -} - -func (d *bincDecDriver) DecodeTime() (t time.Time) { - if !d.bdRead { - d.readNextBd() - } - if d.bd == bincVdSpecial<<4|bincSpNil { - d.bdRead = false - return - } - if d.vd != bincVdTimestamp { - d.d.errorf("cannot decode time - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) - return - } - t, err := bincDecodeTime(d.r.readx(uint(d.vs))) - if err != nil { - panic(err) - } - d.bdRead = false - return -} - -func (d *bincDecDriver) decFloatPre(vs, defaultLen byte) { - if vs&0x8 == 0 { - d.r.readb(d.b[0:defaultLen]) - } else { - l := d.r.readn1() - if l > 8 { - d.d.errorf("cannot read float - at most 8 bytes used to represent float - received %v bytes", l) - return - } - for i := l; i < 8; i++ { - d.b[i] = 0 - } - d.r.readb(d.b[0:l]) - } -} - -func (d *bincDecDriver) decFloat() (f float64) { - //if true { f = math.Float64frombits(bigen.Uint64(d.r.readx(8))); break; } - if x := d.vs & 0x7; x == bincFlBin32 { - d.decFloatPre(d.vs, 4) - f = float64(math.Float32frombits(bigen.Uint32(d.b[0:4]))) - } else if x == bincFlBin64 { - d.decFloatPre(d.vs, 8) - f = math.Float64frombits(bigen.Uint64(d.b[0:8])) - } else { - d.d.errorf("read float - only float32 and float64 are supported - %s %x-%x/%s", - msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) - return - } - return -} - -func (d *bincDecDriver) decUint() (v uint64) { - // need to inline the code (interface conversion and type assertion expensive) - switch d.vs { - case 0: - v = uint64(d.r.readn1()) - case 1: - d.r.readb(d.b[6:8]) - v = uint64(bigen.Uint16(d.b[6:8])) - case 2: - d.b[4] = 0 - d.r.readb(d.b[5:8]) - v = uint64(bigen.Uint32(d.b[4:8])) - case 3: - d.r.readb(d.b[4:8]) - v = uint64(bigen.Uint32(d.b[4:8])) - case 4, 5, 6: - lim := 7 - d.vs - d.r.readb(d.b[lim:8]) - for i := uint8(0); i < lim; i++ { - d.b[i] = 0 - } - v = uint64(bigen.Uint64(d.b[:8])) - case 7: - d.r.readb(d.b[:8]) - v = uint64(bigen.Uint64(d.b[:8])) - default: - d.d.errorf("unsigned integers with greater than 64 bits of precision not supported") - return - } - return -} - -func (d *bincDecDriver) decCheckInteger() (ui uint64, neg bool) { - if !d.bdRead { - d.readNextBd() - } - vd, vs := d.vd, d.vs - if vd == bincVdPosInt { - ui = d.decUint() - } else if vd == bincVdNegInt { - ui = d.decUint() - neg = true - } else if vd == bincVdSmallInt { - ui = uint64(d.vs) + 1 - } else if vd == bincVdSpecial { - if vs == bincSpZero { - //i = 0 - } else if vs == bincSpNegOne { - neg = true - ui = 1 - } else { - d.d.errorf("integer decode fails - invalid special value from descriptor %x-%x/%s", - d.vd, d.vs, bincdesc(d.vd, d.vs)) - return - } - } else { - d.d.errorf("integer can only be decoded from int/uint. d.bd: 0x%x, d.vd: 0x%x", d.bd, d.vd) - return - } - return -} - -func (d *bincDecDriver) DecodeInt64() (i int64) { - ui, neg := d.decCheckInteger() - i = chkOvf.SignedIntV(ui) - if neg { - i = -i - } - d.bdRead = false - return -} - -func (d *bincDecDriver) DecodeUint64() (ui uint64) { - ui, neg := d.decCheckInteger() - if neg { - d.d.errorf("assigning negative signed value to unsigned integer type") - return - } - d.bdRead = false - return -} - -func (d *bincDecDriver) DecodeFloat64() (f float64) { - if !d.bdRead { - d.readNextBd() - } - vd, vs := d.vd, d.vs - if vd == bincVdSpecial { - d.bdRead = false - if vs == bincSpNan { - return math.NaN() - } else if vs == bincSpPosInf { - return math.Inf(1) - } else if vs == bincSpZeroFloat || vs == bincSpZero { - return - } else if vs == bincSpNegInf { - return math.Inf(-1) - } else { - d.d.errorf("float - invalid special value from descriptor %x-%x/%s", - d.vd, d.vs, bincdesc(d.vd, d.vs)) - return - } - } else if vd == bincVdFloat { - f = d.decFloat() - } else { - f = float64(d.DecodeInt64()) - } - d.bdRead = false - return -} - -// bool can be decoded from bool only (single byte). -func (d *bincDecDriver) DecodeBool() (b bool) { - if !d.bdRead { - d.readNextBd() - } - if bd := d.bd; bd == (bincVdSpecial | bincSpFalse) { - // b = false - } else if bd == (bincVdSpecial | bincSpTrue) { - b = true - } else { - d.d.errorf("bool - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) - return - } - d.bdRead = false - return -} - -func (d *bincDecDriver) ReadMapStart() (length int) { - if !d.bdRead { - d.readNextBd() - } - if d.vd != bincVdMap { - d.d.errorf("map - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) - return - } - length = d.decLen() - d.bdRead = false - return -} - -func (d *bincDecDriver) ReadArrayStart() (length int) { - if !d.bdRead { - d.readNextBd() - } - if d.vd != bincVdArray { - d.d.errorf("array - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) - return - } - length = d.decLen() - d.bdRead = false - return -} - -func (d *bincDecDriver) decLen() int { - if d.vs > 3 { - return int(d.vs - 4) - } - return int(d.decLenNumber()) -} - -func (d *bincDecDriver) decLenNumber() (v uint64) { - if x := d.vs; x == 0 { - v = uint64(d.r.readn1()) - } else if x == 1 { - d.r.readb(d.b[6:8]) - v = uint64(bigen.Uint16(d.b[6:8])) - } else if x == 2 { - d.r.readb(d.b[4:8]) - v = uint64(bigen.Uint32(d.b[4:8])) - } else { - d.r.readb(d.b[:8]) - v = bigen.Uint64(d.b[:8]) - } - return -} - -func (d *bincDecDriver) decStringAndBytes(bs []byte, withString, zerocopy bool) ( - bs2 []byte, s string) { - if !d.bdRead { - d.readNextBd() - } - if d.bd == bincVdSpecial<<4|bincSpNil { - d.bdRead = false - return - } - var slen = -1 - // var ok bool - switch d.vd { - case bincVdString, bincVdByteArray: - slen = d.decLen() - if zerocopy { - if d.br { - bs2 = d.r.readx(uint(slen)) - } else if len(bs) == 0 { - bs2 = decByteSlice(d.r, slen, d.d.h.MaxInitLen, d.b[:]) - } else { - bs2 = decByteSlice(d.r, slen, d.d.h.MaxInitLen, bs) - } - } else { - bs2 = decByteSlice(d.r, slen, d.d.h.MaxInitLen, bs) - } - if withString { - s = string(bs2) - } - case bincVdSymbol: - // zerocopy doesn't apply for symbols, - // as the values must be stored in a table for later use. - // - //from vs: extract numSymbolBytes, containsStringVal, strLenPrecision, - //extract symbol - //if containsStringVal, read it and put in map - //else look in map for string value - var symbol uint16 - vs := d.vs - if vs&0x8 == 0 { - symbol = uint16(d.r.readn1()) - } else { - symbol = uint16(bigen.Uint16(d.r.readx(2))) - } - if d.s == nil { - d.s = make([]bincDecSymbol, 0, 16) - } - - if vs&0x4 == 0 { - for i := range d.s { - j := &d.s[i] - if j.i == symbol { - bs2 = j.b - if withString { - if j.s == "" && bs2 != nil { - j.s = string(bs2) - } - s = j.s - } - break - } - } - } else { - switch vs & 0x3 { - case 0: - slen = int(d.r.readn1()) - case 1: - slen = int(bigen.Uint16(d.r.readx(2))) - case 2: - slen = int(bigen.Uint32(d.r.readx(4))) - case 3: - slen = int(bigen.Uint64(d.r.readx(8))) - } - // since using symbols, do not store any part of - // the parameter bs in the map, as it might be a shared buffer. - // bs2 = decByteSlice(d.r, slen, bs) - bs2 = decByteSlice(d.r, slen, d.d.h.MaxInitLen, nil) - if withString { - s = string(bs2) - } - d.s = append(d.s, bincDecSymbol{i: symbol, s: s, b: bs2}) - } - default: - d.d.errorf("string/bytes - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) - return - } - d.bdRead = false - return -} - -func (d *bincDecDriver) DecodeString() (s string) { - // DecodeBytes does not accommodate symbols, whose impl stores string version in map. - // Use decStringAndBytes directly. - // return string(d.DecodeBytes(d.b[:], true, true)) - _, s = d.decStringAndBytes(d.b[:], true, true) - return -} - -func (d *bincDecDriver) DecodeStringAsBytes() (s []byte) { - s, _ = d.decStringAndBytes(d.b[:], false, true) - return -} - -func (d *bincDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) { - if !d.bdRead { - d.readNextBd() - } - if d.bd == bincVdSpecial<<4|bincSpNil { - d.bdRead = false - return nil - } - // check if an "array" of uint8's (see ContainerType for how to infer if an array) - if d.vd == bincVdArray { - bsOut, _ = fastpathTV.DecSliceUint8V(bs, true, d.d) - return - } - var clen int - if d.vd == bincVdString || d.vd == bincVdByteArray { - clen = d.decLen() - } else { - d.d.errorf("bytes - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) - return - } - d.bdRead = false - if zerocopy { - if d.br { - return d.r.readx(uint(clen)) - } else if len(bs) == 0 { - bs = d.b[:] - } - } - return decByteSlice(d.r, clen, d.d.h.MaxInitLen, bs) -} - -func (d *bincDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) { - if xtag > 0xff { - d.d.errorf("ext: tag must be <= 0xff; got: %v", xtag) - return - } - realxtag1, xbs := d.decodeExtV(ext != nil, uint8(xtag)) - realxtag = uint64(realxtag1) - if ext == nil { - re := rv.(*RawExt) - re.Tag = realxtag - re.Data = detachZeroCopyBytes(d.br, re.Data, xbs) - } else { - ext.ReadExt(rv, xbs) - } - return -} - -func (d *bincDecDriver) decodeExtV(verifyTag bool, tag byte) (xtag byte, xbs []byte) { - if !d.bdRead { - d.readNextBd() - } - if d.vd == bincVdCustomExt { - l := d.decLen() - xtag = d.r.readn1() - if verifyTag && xtag != tag { - d.d.errorf("wrong extension tag - got %b, expecting: %v", xtag, tag) - return - } - if d.br { - xbs = d.r.readx(uint(l)) - } else { - xbs = decByteSlice(d.r, l, d.d.h.MaxInitLen, d.d.b[:]) - } - } else if d.vd == bincVdByteArray { - xbs = d.DecodeBytes(nil, true) - } else { - d.d.errorf("ext - expecting extensions or byte array - %s %x-%x/%s", - msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) - return - } - d.bdRead = false - return -} - -func (d *bincDecDriver) DecodeNaked() { - if !d.bdRead { - d.readNextBd() - } - - n := d.d.naked() - var decodeFurther bool - - switch d.vd { - case bincVdSpecial: - switch d.vs { - case bincSpNil: - n.v = valueTypeNil - case bincSpFalse: - n.v = valueTypeBool - n.b = false - case bincSpTrue: - n.v = valueTypeBool - n.b = true - case bincSpNan: - n.v = valueTypeFloat - n.f = math.NaN() - case bincSpPosInf: - n.v = valueTypeFloat - n.f = math.Inf(1) - case bincSpNegInf: - n.v = valueTypeFloat - n.f = math.Inf(-1) - case bincSpZeroFloat: - n.v = valueTypeFloat - n.f = float64(0) - case bincSpZero: - n.v = valueTypeUint - n.u = uint64(0) // int8(0) - case bincSpNegOne: - n.v = valueTypeInt - n.i = int64(-1) // int8(-1) - default: - d.d.errorf("cannot infer value - unrecognized special value from descriptor %x-%x/%s", - d.vd, d.vs, bincdesc(d.vd, d.vs)) - } - case bincVdSmallInt: - n.v = valueTypeUint - n.u = uint64(int8(d.vs)) + 1 // int8(d.vs) + 1 - case bincVdPosInt: - n.v = valueTypeUint - n.u = d.decUint() - case bincVdNegInt: - n.v = valueTypeInt - n.i = -(int64(d.decUint())) - case bincVdFloat: - n.v = valueTypeFloat - n.f = d.decFloat() - case bincVdSymbol: - n.v = valueTypeSymbol - n.s = d.DecodeString() - case bincVdString: - n.v = valueTypeString - n.s = d.DecodeString() - case bincVdByteArray: - decNakedReadRawBytes(d, d.d, n, d.h.RawToString) - case bincVdTimestamp: - n.v = valueTypeTime - tt, err := bincDecodeTime(d.r.readx(uint(d.vs))) - if err != nil { - panic(err) - } - n.t = tt - case bincVdCustomExt: - n.v = valueTypeExt - l := d.decLen() - n.u = uint64(d.r.readn1()) - if d.br { - n.l = d.r.readx(uint(l)) - } else { - n.l = decByteSlice(d.r, l, d.d.h.MaxInitLen, d.d.b[:]) - } - case bincVdArray: - n.v = valueTypeArray - decodeFurther = true - case bincVdMap: - n.v = valueTypeMap - decodeFurther = true - default: - d.d.errorf("cannot infer value - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs)) - } - - if !decodeFurther { - d.bdRead = false - } - if n.v == valueTypeUint && d.h.SignedInteger { - n.v = valueTypeInt - n.i = int64(n.u) - } -} - -//------------------------------------ - -// BincHandle is a Handle for the Binc Schema-Free Encoding Format -// defined at https://github.com/ugorji/binc . -// -// BincHandle currently supports all Binc features with the following EXCEPTIONS: -// - only integers up to 64 bits of precision are supported. -// big integers are unsupported. -// - Only IEEE 754 binary32 and binary64 floats are supported (ie Go float32 and float64 types). -// extended precision and decimal IEEE 754 floats are unsupported. -// - Only UTF-8 strings supported. -// Unicode_Other Binc types (UTF16, UTF32) are currently unsupported. -// -// Note that these EXCEPTIONS are temporary and full support is possible and may happen soon. -type BincHandle struct { - BasicHandle - binaryEncodingType - noElemSeparators - - // AsSymbols defines what should be encoded as symbols. - // - // Encoding as symbols can reduce the encoded size significantly. - // - // However, during decoding, each string to be encoded as a symbol must - // be checked to see if it has been seen before. Consequently, encoding time - // will increase if using symbols, because string comparisons has a clear cost. - // - // Values: - // - 0: default: library uses best judgement - // - 1: use symbols - // - 2: do not use symbols - AsSymbols uint8 - - // AsSymbols: may later on introduce more options ... - // - m: map keys - // - s: struct fields - // - n: none - // - a: all: same as m, s, ... - - // _ [1]uint64 // padding -} - -// Name returns the name of the handle: binc -func (h *BincHandle) Name() string { return "binc" } - -// SetBytesExt sets an extension -func (h *BincHandle) SetBytesExt(rt reflect.Type, tag uint64, ext BytesExt) (err error) { - return h.SetExt(rt, tag, &extWrapper{ext, interfaceExtFailer{}}) -} - -func (h *BincHandle) newEncDriver(e *Encoder) encDriver { - return &bincEncDriver{e: e, h: h, w: e.w} -} - -func (h *BincHandle) newDecDriver(d *Decoder) decDriver { - return &bincDecDriver{d: d, h: h, r: d.r, br: d.bytes} -} - -func (e *bincEncDriver) reset() { - e.w = e.e.w - e.s = 0 - e.c = 0 - e.m = nil -} - -func (d *bincDecDriver) reset() { - d.r, d.br = d.d.r, d.d.bytes - d.s = nil - d.bd, d.bdRead, d.vd, d.vs = 0, false, 0, 0 -} - -// var timeDigits = [...]byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'} - -// EncodeTime encodes a time.Time as a []byte, including -// information on the instant in time and UTC offset. -// -// Format Description -// -// A timestamp is composed of 3 components: -// -// - secs: signed integer representing seconds since unix epoch -// - nsces: unsigned integer representing fractional seconds as a -// nanosecond offset within secs, in the range 0 <= nsecs < 1e9 -// - tz: signed integer representing timezone offset in minutes east of UTC, -// and a dst (daylight savings time) flag -// -// When encoding a timestamp, the first byte is the descriptor, which -// defines which components are encoded and how many bytes are used to -// encode secs and nsecs components. *If secs/nsecs is 0 or tz is UTC, it -// is not encoded in the byte array explicitly*. -// -// Descriptor 8 bits are of the form `A B C DDD EE`: -// A: Is secs component encoded? 1 = true -// B: Is nsecs component encoded? 1 = true -// C: Is tz component encoded? 1 = true -// DDD: Number of extra bytes for secs (range 0-7). -// If A = 1, secs encoded in DDD+1 bytes. -// If A = 0, secs is not encoded, and is assumed to be 0. -// If A = 1, then we need at least 1 byte to encode secs. -// DDD says the number of extra bytes beyond that 1. -// E.g. if DDD=0, then secs is represented in 1 byte. -// if DDD=2, then secs is represented in 3 bytes. -// EE: Number of extra bytes for nsecs (range 0-3). -// If B = 1, nsecs encoded in EE+1 bytes (similar to secs/DDD above) -// -// Following the descriptor bytes, subsequent bytes are: -// -// secs component encoded in `DDD + 1` bytes (if A == 1) -// nsecs component encoded in `EE + 1` bytes (if B == 1) -// tz component encoded in 2 bytes (if C == 1) -// -// secs and nsecs components are integers encoded in a BigEndian -// 2-complement encoding format. -// -// tz component is encoded as 2 bytes (16 bits). Most significant bit 15 to -// Least significant bit 0 are described below: -// -// Timezone offset has a range of -12:00 to +14:00 (ie -720 to +840 minutes). -// Bit 15 = have\_dst: set to 1 if we set the dst flag. -// Bit 14 = dst\_on: set to 1 if dst is in effect at the time, or 0 if not. -// Bits 13..0 = timezone offset in minutes. It is a signed integer in Big Endian format. -func bincEncodeTime(t time.Time) []byte { - //t := rv.Interface().(time.Time) - tsecs, tnsecs := t.Unix(), t.Nanosecond() - var ( - bd byte - btmp [8]byte - bs [16]byte - i int = 1 - ) - l := t.Location() - if l == time.UTC { - l = nil - } - if tsecs != 0 { - bd = bd | 0x80 - bigen.PutUint64(btmp[:], uint64(tsecs)) - f := pruneSignExt(btmp[:], tsecs >= 0) - bd = bd | (byte(7-f) << 2) - copy(bs[i:], btmp[f:]) - i = i + (8 - f) - } - if tnsecs != 0 { - bd = bd | 0x40 - bigen.PutUint32(btmp[:4], uint32(tnsecs)) - f := pruneSignExt(btmp[:4], true) - bd = bd | byte(3-f) - copy(bs[i:], btmp[f:4]) - i = i + (4 - f) - } - if l != nil { - bd = bd | 0x20 - // Note that Go Libs do not give access to dst flag. - _, zoneOffset := t.Zone() - //zoneName, zoneOffset := t.Zone() - zoneOffset /= 60 - z := uint16(zoneOffset) - bigen.PutUint16(btmp[:2], z) - // clear dst flags - bs[i] = btmp[0] & 0x3f - bs[i+1] = btmp[1] - i = i + 2 - } - bs[0] = bd - return bs[0:i] -} - -// bincDecodeTime decodes a []byte into a time.Time. -func bincDecodeTime(bs []byte) (tt time.Time, err error) { - bd := bs[0] - var ( - tsec int64 - tnsec uint32 - tz uint16 - i byte = 1 - i2 byte - n byte - ) - if bd&(1<<7) != 0 { - var btmp [8]byte - n = ((bd >> 2) & 0x7) + 1 - i2 = i + n - copy(btmp[8-n:], bs[i:i2]) - //if first bit of bs[i] is set, then fill btmp[0..8-n] with 0xff (ie sign extend it) - if bs[i]&(1<<7) != 0 { - copy(btmp[0:8-n], bsAll0xff) - //for j,k := byte(0), 8-n; j < k; j++ { btmp[j] = 0xff } - } - i = i2 - tsec = int64(bigen.Uint64(btmp[:])) - } - if bd&(1<<6) != 0 { - var btmp [4]byte - n = (bd & 0x3) + 1 - i2 = i + n - copy(btmp[4-n:], bs[i:i2]) - i = i2 - tnsec = bigen.Uint32(btmp[:]) - } - if bd&(1<<5) == 0 { - tt = time.Unix(tsec, int64(tnsec)).UTC() - return - } - // In stdlib time.Parse, when a date is parsed without a zone name, it uses "" as zone name. - // However, we need name here, so it can be shown when time is printf.d. - // Zone name is in form: UTC-08:00. - // Note that Go Libs do not give access to dst flag, so we ignore dst bits - - i2 = i + 2 - tz = bigen.Uint16(bs[i:i2]) - // i = i2 - // sign extend sign bit into top 2 MSB (which were dst bits): - if tz&(1<<13) == 0 { // positive - tz = tz & 0x3fff //clear 2 MSBs: dst bits - } else { // negative - tz = tz | 0xc000 //set 2 MSBs: dst bits - } - tzint := int16(tz) - if tzint == 0 { - tt = time.Unix(tsec, int64(tnsec)).UTC() - } else { - // For Go Time, do not use a descriptive timezone. - // It's unnecessary, and makes it harder to do a reflect.DeepEqual. - // The Offset already tells what the offset should be, if not on UTC and unknown zone name. - // var zoneName = timeLocUTCName(tzint) - tt = time.Unix(tsec, int64(tnsec)).In(time.FixedZone("", int(tzint)*60)) - } - return -} - -var _ decDriver = (*bincDecDriver)(nil) -var _ encDriver = (*bincEncDriver)(nil) diff --git a/codec/cbor.go b/codec/cbor.go deleted file mode 100644 index 7833f9d6..00000000 --- a/codec/cbor.go +++ /dev/null @@ -1,767 +0,0 @@ -// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -package codec - -import ( - "math" - "reflect" - "time" -) - -const ( - cborMajorUint byte = iota - cborMajorNegInt - cborMajorBytes - cborMajorText - cborMajorArray - cborMajorMap - cborMajorTag - cborMajorOther -) - -const ( - cborBdFalse byte = 0xf4 + iota - cborBdTrue - cborBdNil - cborBdUndefined - cborBdExt - cborBdFloat16 - cborBdFloat32 - cborBdFloat64 -) - -const ( - cborBdIndefiniteBytes byte = 0x5f - cborBdIndefiniteString byte = 0x7f - cborBdIndefiniteArray byte = 0x9f - cborBdIndefiniteMap byte = 0xbf - cborBdBreak byte = 0xff -) - -// These define some in-stream descriptors for -// manual encoding e.g. when doing explicit indefinite-length -const ( - CborStreamBytes byte = 0x5f - CborStreamString byte = 0x7f - CborStreamArray byte = 0x9f - CborStreamMap byte = 0xbf - CborStreamBreak byte = 0xff -) - -const ( - cborBaseUint byte = 0x00 - cborBaseNegInt byte = 0x20 - cborBaseBytes byte = 0x40 - cborBaseString byte = 0x60 - cborBaseArray byte = 0x80 - cborBaseMap byte = 0xa0 - cborBaseTag byte = 0xc0 - cborBaseSimple byte = 0xe0 -) - -func cbordesc(bd byte) string { - switch bd { - case cborBdNil: - return "nil" - case cborBdFalse: - return "false" - case cborBdTrue: - return "true" - case cborBdFloat16, cborBdFloat32, cborBdFloat64: - return "float" - case cborBdIndefiniteBytes: - return "bytes*" - case cborBdIndefiniteString: - return "string*" - case cborBdIndefiniteArray: - return "array*" - case cborBdIndefiniteMap: - return "map*" - default: - switch { - case bd >= cborBaseUint && bd < cborBaseNegInt: - return "(u)int" - case bd >= cborBaseNegInt && bd < cborBaseBytes: - return "int" - case bd >= cborBaseBytes && bd < cborBaseString: - return "bytes" - case bd >= cborBaseString && bd < cborBaseArray: - return "string" - case bd >= cborBaseArray && bd < cborBaseMap: - return "array" - case bd >= cborBaseMap && bd < cborBaseTag: - return "map" - case bd >= cborBaseTag && bd < cborBaseSimple: - return "ext" - default: - return "unknown" - } - } -} - -// ------------------- - -type cborEncDriver struct { - noBuiltInTypes - encDriverNoopContainerWriter - e *Encoder - w *encWriterSwitch - h *CborHandle - x [8]byte - // _ [3]uint64 // padding -} - -func (e *cborEncDriver) EncodeNil() { - e.w.writen1(cborBdNil) -} - -func (e *cborEncDriver) EncodeBool(b bool) { - if b { - e.w.writen1(cborBdTrue) - } else { - e.w.writen1(cborBdFalse) - } -} - -func (e *cborEncDriver) EncodeFloat32(f float32) { - e.w.writen1(cborBdFloat32) - bigenHelper{e.x[:4], e.w}.writeUint32(math.Float32bits(f)) -} - -func (e *cborEncDriver) EncodeFloat64(f float64) { - e.w.writen1(cborBdFloat64) - bigenHelper{e.x[:8], e.w}.writeUint64(math.Float64bits(f)) -} - -func (e *cborEncDriver) encUint(v uint64, bd byte) { - if v <= 0x17 { - e.w.writen1(byte(v) + bd) - } else if v <= math.MaxUint8 { - e.w.writen2(bd+0x18, uint8(v)) - } else if v <= math.MaxUint16 { - e.w.writen1(bd + 0x19) - bigenHelper{e.x[:2], e.w}.writeUint16(uint16(v)) - } else if v <= math.MaxUint32 { - e.w.writen1(bd + 0x1a) - bigenHelper{e.x[:4], e.w}.writeUint32(uint32(v)) - } else { // if v <= math.MaxUint64 { - e.w.writen1(bd + 0x1b) - bigenHelper{e.x[:8], e.w}.writeUint64(v) - } -} - -func (e *cborEncDriver) EncodeInt(v int64) { - if v < 0 { - e.encUint(uint64(-1-v), cborBaseNegInt) - } else { - e.encUint(uint64(v), cborBaseUint) - } -} - -func (e *cborEncDriver) EncodeUint(v uint64) { - e.encUint(v, cborBaseUint) -} - -func (e *cborEncDriver) encLen(bd byte, length int) { - e.encUint(uint64(length), bd) -} - -func (e *cborEncDriver) EncodeTime(t time.Time) { - if t.IsZero() { - e.EncodeNil() - } else if e.h.TimeRFC3339 { - e.encUint(0, cborBaseTag) - e.EncodeStringEnc(cUTF8, t.Format(time.RFC3339Nano)) - } else { - e.encUint(1, cborBaseTag) - t = t.UTC().Round(time.Microsecond) - sec, nsec := t.Unix(), uint64(t.Nanosecond()) - if nsec == 0 { - e.EncodeInt(sec) - } else { - e.EncodeFloat64(float64(sec) + float64(nsec)/1e9) - } - } -} - -func (e *cborEncDriver) EncodeExt(rv interface{}, xtag uint64, ext Ext, en *Encoder) { - e.encUint(uint64(xtag), cborBaseTag) - if v := ext.ConvertExt(rv); v == nil { - e.EncodeNil() - } else { - en.encode(v) - } -} - -func (e *cborEncDriver) EncodeRawExt(re *RawExt, en *Encoder) { - e.encUint(uint64(re.Tag), cborBaseTag) - // only encodes re.Value (never re.Data) - // if false && re.Data != nil { - // en.encode(re.Data) - // } else if re.Value != nil { - if re.Value != nil { - en.encode(re.Value) - } else { - e.EncodeNil() - } -} - -func (e *cborEncDriver) WriteArrayStart(length int) { - if e.h.IndefiniteLength { - e.w.writen1(cborBdIndefiniteArray) - } else { - e.encLen(cborBaseArray, length) - } -} - -func (e *cborEncDriver) WriteMapStart(length int) { - if e.h.IndefiniteLength { - e.w.writen1(cborBdIndefiniteMap) - } else { - e.encLen(cborBaseMap, length) - } -} - -func (e *cborEncDriver) WriteMapEnd() { - if e.h.IndefiniteLength { - e.w.writen1(cborBdBreak) - } -} - -func (e *cborEncDriver) WriteArrayEnd() { - if e.h.IndefiniteLength { - e.w.writen1(cborBdBreak) - } -} - -func (e *cborEncDriver) EncodeString(c charEncoding, v string) { - e.encStringBytesS(cborBaseString, v) -} - -func (e *cborEncDriver) EncodeStringEnc(c charEncoding, v string) { - e.encStringBytesS(cborBaseString, v) -} - -func (e *cborEncDriver) EncodeStringBytes(c charEncoding, v []byte) { - if v == nil { - e.EncodeNil() - } else if c == cRAW { - e.encStringBytesS(cborBaseBytes, stringView(v)) - } else { - e.encStringBytesS(cborBaseString, stringView(v)) - } -} - -func (e *cborEncDriver) EncodeStringBytesRaw(v []byte) { - if v == nil { - e.EncodeNil() - } else { - e.encStringBytesS(cborBaseBytes, stringView(v)) - } -} - -func (e *cborEncDriver) encStringBytesS(bb byte, v string) { - if e.h.IndefiniteLength { - if bb == cborBaseBytes { - e.w.writen1(cborBdIndefiniteBytes) - } else { - e.w.writen1(cborBdIndefiniteString) - } - var vlen uint = uint(len(v)) - blen := vlen / 4 - if blen == 0 { - blen = 64 - } else if blen > 1024 { - blen = 1024 - } - for i := uint(0); i < vlen; { - var v2 string - i2 := i + blen - if i2 < vlen { - v2 = v[i:i2] - } else { - v2 = v[i:] - } - e.encLen(bb, len(v2)) - e.w.writestr(v2) - i = i2 - } - e.w.writen1(cborBdBreak) - } else { - e.encLen(bb, len(v)) - e.w.writestr(v) - } -} - -// ---------------------- - -type cborDecDriver struct { - d *Decoder - h *CborHandle - r *decReaderSwitch - br bool // bytes reader - bdRead bool - bd byte - noBuiltInTypes - // decNoSeparator - decDriverNoopContainerReader - // _ [3]uint64 // padding -} - -func (d *cborDecDriver) readNextBd() { - d.bd = d.r.readn1() - d.bdRead = true -} - -func (d *cborDecDriver) uncacheRead() { - if d.bdRead { - d.r.unreadn1() - d.bdRead = false - } -} - -func (d *cborDecDriver) ContainerType() (vt valueType) { - if !d.bdRead { - d.readNextBd() - } - if d.bd == cborBdNil { - return valueTypeNil - } else if d.bd == cborBdIndefiniteBytes || (d.bd >= cborBaseBytes && d.bd < cborBaseString) { - return valueTypeBytes - } else if d.bd == cborBdIndefiniteString || (d.bd >= cborBaseString && d.bd < cborBaseArray) { - return valueTypeString - } else if d.bd == cborBdIndefiniteArray || (d.bd >= cborBaseArray && d.bd < cborBaseMap) { - return valueTypeArray - } else if d.bd == cborBdIndefiniteMap || (d.bd >= cborBaseMap && d.bd < cborBaseTag) { - return valueTypeMap - } - // else { - // d.d.errorf("isContainerType: unsupported parameter: %v", vt) - // } - return valueTypeUnset -} - -func (d *cborDecDriver) TryDecodeAsNil() bool { - if !d.bdRead { - d.readNextBd() - } - // treat Nil and Undefined as nil values - if d.bd == cborBdNil || d.bd == cborBdUndefined { - d.bdRead = false - return true - } - return false -} - -func (d *cborDecDriver) CheckBreak() bool { - if !d.bdRead { - d.readNextBd() - } - if d.bd == cborBdBreak { - d.bdRead = false - return true - } - return false -} - -func (d *cborDecDriver) decUint() (ui uint64) { - v := d.bd & 0x1f - if v <= 0x17 { - ui = uint64(v) - } else { - if v == 0x18 { - ui = uint64(d.r.readn1()) - } else if v == 0x19 { - ui = uint64(bigen.Uint16(d.r.readx(2))) - } else if v == 0x1a { - ui = uint64(bigen.Uint32(d.r.readx(4))) - } else if v == 0x1b { - ui = uint64(bigen.Uint64(d.r.readx(8))) - } else { - d.d.errorf("invalid descriptor decoding uint: %x/%s", d.bd, cbordesc(d.bd)) - return - } - } - return -} - -func (d *cborDecDriver) decCheckInteger() (neg bool) { - if !d.bdRead { - d.readNextBd() - } - major := d.bd >> 5 - if major == cborMajorUint { - } else if major == cborMajorNegInt { - neg = true - } else { - d.d.errorf("not an integer - invalid major %v from descriptor %x/%s", - major, d.bd, cbordesc(d.bd)) - return - } - return -} - -func (d *cborDecDriver) DecodeInt64() (i int64) { - neg := d.decCheckInteger() - ui := d.decUint() - // check if this number can be converted to an int without overflow - if neg { - i = -(chkOvf.SignedIntV(ui + 1)) - } else { - i = chkOvf.SignedIntV(ui) - } - d.bdRead = false - return -} - -func (d *cborDecDriver) DecodeUint64() (ui uint64) { - if d.decCheckInteger() { - d.d.errorf("assigning negative signed value to unsigned type") - return - } - ui = d.decUint() - d.bdRead = false - return -} - -func (d *cborDecDriver) DecodeFloat64() (f float64) { - if !d.bdRead { - d.readNextBd() - } - if bd := d.bd; bd == cborBdFloat16 { - f = float64(math.Float32frombits(halfFloatToFloatBits(bigen.Uint16(d.r.readx(2))))) - } else if bd == cborBdFloat32 { - f = float64(math.Float32frombits(bigen.Uint32(d.r.readx(4)))) - } else if bd == cborBdFloat64 { - f = math.Float64frombits(bigen.Uint64(d.r.readx(8))) - } else if bd >= cborBaseUint && bd < cborBaseBytes { - f = float64(d.DecodeInt64()) - } else { - d.d.errorf("float only valid from float16/32/64 - invalid descriptor %x/%s", bd, cbordesc(bd)) - return - } - d.bdRead = false - return -} - -// bool can be decoded from bool only (single byte). -func (d *cborDecDriver) DecodeBool() (b bool) { - if !d.bdRead { - d.readNextBd() - } - if bd := d.bd; bd == cborBdTrue { - b = true - } else if bd == cborBdFalse { - } else { - d.d.errorf("not bool - %s %x/%s", msgBadDesc, d.bd, cbordesc(d.bd)) - return - } - d.bdRead = false - return -} - -func (d *cborDecDriver) ReadMapStart() (length int) { - if !d.bdRead { - d.readNextBd() - } - d.bdRead = false - if d.bd == cborBdIndefiniteMap { - return -1 - } - return d.decLen() -} - -func (d *cborDecDriver) ReadArrayStart() (length int) { - if !d.bdRead { - d.readNextBd() - } - d.bdRead = false - if d.bd == cborBdIndefiniteArray { - return -1 - } - return d.decLen() -} - -func (d *cborDecDriver) decLen() int { - return int(d.decUint()) -} - -func (d *cborDecDriver) decAppendIndefiniteBytes(bs []byte) []byte { - d.bdRead = false - for { - if d.CheckBreak() { - break - } - if major := d.bd >> 5; major != cborMajorBytes && major != cborMajorText { - d.d.errorf("expect bytes/string major type in indefinite string/bytes;"+ - " got major %v from descriptor %x/%x", major, d.bd, cbordesc(d.bd)) - return nil - } - n := d.decLen() - oldLen := len(bs) - newLen := oldLen + n - if newLen > cap(bs) { - bs2 := make([]byte, newLen, 2*cap(bs)+n) - copy(bs2, bs) - bs = bs2 - } else { - bs = bs[:newLen] - } - d.r.readb(bs[oldLen:newLen]) - // bs = append(bs, d.r.readn()...) - d.bdRead = false - } - d.bdRead = false - return bs -} - -func (d *cborDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) { - if !d.bdRead { - d.readNextBd() - } - if d.bd == cborBdNil || d.bd == cborBdUndefined { - d.bdRead = false - return nil - } - if d.bd == cborBdIndefiniteBytes || d.bd == cborBdIndefiniteString { - d.bdRead = false - if bs == nil { - if zerocopy { - return d.decAppendIndefiniteBytes(d.d.b[:0]) - } - return d.decAppendIndefiniteBytes(zeroByteSlice) - } - return d.decAppendIndefiniteBytes(bs[:0]) - } - // check if an "array" of uint8's (see ContainerType for how to infer if an array) - if d.bd == cborBdIndefiniteArray || (d.bd >= cborBaseArray && d.bd < cborBaseMap) { - bsOut, _ = fastpathTV.DecSliceUint8V(bs, true, d.d) - return - } - clen := d.decLen() - d.bdRead = false - if zerocopy { - if d.br { - return d.r.readx(uint(clen)) - } else if len(bs) == 0 { - bs = d.d.b[:] - } - } - return decByteSlice(d.r, clen, d.h.MaxInitLen, bs) -} - -func (d *cborDecDriver) DecodeString() (s string) { - return string(d.DecodeBytes(d.d.b[:], true)) -} - -func (d *cborDecDriver) DecodeStringAsBytes() (s []byte) { - return d.DecodeBytes(d.d.b[:], true) -} - -func (d *cborDecDriver) DecodeTime() (t time.Time) { - if !d.bdRead { - d.readNextBd() - } - if d.bd == cborBdNil || d.bd == cborBdUndefined { - d.bdRead = false - return - } - xtag := d.decUint() - d.bdRead = false - return d.decodeTime(xtag) -} - -func (d *cborDecDriver) decodeTime(xtag uint64) (t time.Time) { - if !d.bdRead { - d.readNextBd() - } - switch xtag { - case 0: - var err error - if t, err = time.Parse(time.RFC3339, stringView(d.DecodeStringAsBytes())); err != nil { - d.d.errorv(err) - } - case 1: - // decode an int64 or a float, and infer time.Time from there. - // for floats, round to microseconds, as that is what is guaranteed to fit well. - switch { - case d.bd == cborBdFloat16, d.bd == cborBdFloat32: - f1, f2 := math.Modf(d.DecodeFloat64()) - t = time.Unix(int64(f1), int64(f2*1e9)) - case d.bd == cborBdFloat64: - f1, f2 := math.Modf(d.DecodeFloat64()) - t = time.Unix(int64(f1), int64(f2*1e9)) - case d.bd >= cborBaseUint && d.bd < cborBaseNegInt, - d.bd >= cborBaseNegInt && d.bd < cborBaseBytes: - t = time.Unix(d.DecodeInt64(), 0) - default: - d.d.errorf("time.Time can only be decoded from a number (or RFC3339 string)") - } - default: - d.d.errorf("invalid tag for time.Time - expecting 0 or 1, got 0x%x", xtag) - } - t = t.UTC().Round(time.Microsecond) - return -} - -func (d *cborDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) { - if !d.bdRead { - d.readNextBd() - } - u := d.decUint() - d.bdRead = false - realxtag = u - if ext == nil { - re := rv.(*RawExt) - re.Tag = realxtag - d.d.decode(&re.Value) - } else if xtag != realxtag { - d.d.errorf("Wrong extension tag. Got %b. Expecting: %v", realxtag, xtag) - return - } else { - var v interface{} - d.d.decode(&v) - ext.UpdateExt(rv, v) - } - d.bdRead = false - return -} - -func (d *cborDecDriver) DecodeNaked() { - if !d.bdRead { - d.readNextBd() - } - - n := d.d.naked() - var decodeFurther bool - - switch d.bd { - case cborBdNil: - n.v = valueTypeNil - case cborBdFalse: - n.v = valueTypeBool - n.b = false - case cborBdTrue: - n.v = valueTypeBool - n.b = true - case cborBdFloat16, cborBdFloat32, cborBdFloat64: - n.v = valueTypeFloat - n.f = d.DecodeFloat64() - case cborBdIndefiniteBytes: - decNakedReadRawBytes(d, d.d, n, d.h.RawToString) - case cborBdIndefiniteString: - n.v = valueTypeString - n.s = d.DecodeString() - case cborBdIndefiniteArray: - n.v = valueTypeArray - decodeFurther = true - case cborBdIndefiniteMap: - n.v = valueTypeMap - decodeFurther = true - default: - switch { - case d.bd >= cborBaseUint && d.bd < cborBaseNegInt: - if d.h.SignedInteger { - n.v = valueTypeInt - n.i = d.DecodeInt64() - } else { - n.v = valueTypeUint - n.u = d.DecodeUint64() - } - case d.bd >= cborBaseNegInt && d.bd < cborBaseBytes: - n.v = valueTypeInt - n.i = d.DecodeInt64() - case d.bd >= cborBaseBytes && d.bd < cborBaseString: - decNakedReadRawBytes(d, d.d, n, d.h.RawToString) - case d.bd >= cborBaseString && d.bd < cborBaseArray: - n.v = valueTypeString - n.s = d.DecodeString() - case d.bd >= cborBaseArray && d.bd < cborBaseMap: - n.v = valueTypeArray - decodeFurther = true - case d.bd >= cborBaseMap && d.bd < cborBaseTag: - n.v = valueTypeMap - decodeFurther = true - case d.bd >= cborBaseTag && d.bd < cborBaseSimple: - n.v = valueTypeExt - n.u = d.decUint() - n.l = nil - if n.u == 0 || n.u == 1 { - d.bdRead = false - n.v = valueTypeTime - n.t = d.decodeTime(n.u) - } - // d.bdRead = false - // d.d.decode(&re.Value) // handled by decode itself. - // decodeFurther = true - default: - d.d.errorf("decodeNaked: Unrecognized d.bd: 0x%x", d.bd) - return - } - } - - if !decodeFurther { - d.bdRead = false - } -} - -// ------------------------- - -// CborHandle is a Handle for the CBOR encoding format, -// defined at http://tools.ietf.org/html/rfc7049 and documented further at http://cbor.io . -// -// CBOR is comprehensively supported, including support for: -// - indefinite-length arrays/maps/bytes/strings -// - (extension) tags in range 0..0xffff (0 .. 65535) -// - half, single and double-precision floats -// - all numbers (1, 2, 4 and 8-byte signed and unsigned integers) -// - nil, true, false, ... -// - arrays and maps, bytes and text strings -// -// None of the optional extensions (with tags) defined in the spec are supported out-of-the-box. -// Users can implement them as needed (using SetExt), including spec-documented ones: -// - timestamp, BigNum, BigFloat, Decimals, -// - Encoded Text (e.g. URL, regexp, base64, MIME Message), etc. -type CborHandle struct { - binaryEncodingType - noElemSeparators - BasicHandle - - // IndefiniteLength=true, means that we encode using indefinitelength - IndefiniteLength bool - - // TimeRFC3339 says to encode time.Time using RFC3339 format. - // If unset, we encode time.Time using seconds past epoch. - TimeRFC3339 bool - - // _ [1]uint64 // padding -} - -// Name returns the name of the handle: cbor -func (h *CborHandle) Name() string { return "cbor" } - -// SetInterfaceExt sets an extension -func (h *CborHandle) SetInterfaceExt(rt reflect.Type, tag uint64, ext InterfaceExt) (err error) { - return h.SetExt(rt, tag, &extWrapper{bytesExtFailer{}, ext}) -} - -func (h *CborHandle) newEncDriver(e *Encoder) encDriver { - return &cborEncDriver{e: e, w: e.w, h: h} -} - -func (h *CborHandle) newDecDriver(d *Decoder) decDriver { - return &cborDecDriver{d: d, h: h, r: d.r, br: d.bytes} -} - -func (e *cborEncDriver) reset() { - e.w = e.e.w -} - -func (d *cborDecDriver) reset() { - d.r, d.br = d.d.r, d.d.bytes - d.bd, d.bdRead = 0, false -} - -var _ decDriver = (*cborDecDriver)(nil) -var _ encDriver = (*cborEncDriver)(nil) diff --git a/codec/cbor_test.go b/codec/cbor_test.go deleted file mode 100644 index 12f5c810..00000000 --- a/codec/cbor_test.go +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -package codec - -import ( - "bufio" - "bytes" - "encoding/hex" - "math" - "os" - "regexp" - "strings" - "testing" -) - -func TestCborIndefiniteLength(t *testing.T) { - oldMapType := testCborH.MapType - defer func() { - testCborH.MapType = oldMapType - }() - testCborH.MapType = testMapStrIntfTyp - // var ( - // M1 map[string][]byte - // M2 map[uint64]bool - // L1 []interface{} - // S1 []string - // B1 []byte - // ) - var v, vv interface{} - // define it (v), encode it using indefinite lengths, decode it (vv), compare v to vv - v = map[string]interface{}{ - "one-byte-key": []byte{1, 2, 3, 4, 5, 6}, - "two-string-key": "two-value", - "three-list-key": []interface{}{true, false, uint64(1), int64(-1)}, - } - var buf bytes.Buffer - // buf.Reset() - e := NewEncoder(&buf, testCborH) - buf.WriteByte(cborBdIndefiniteMap) - //---- - buf.WriteByte(cborBdIndefiniteString) - e.MustEncode("one-") - e.MustEncode("byte-") - e.MustEncode("key") - buf.WriteByte(cborBdBreak) - - buf.WriteByte(cborBdIndefiniteBytes) - e.MustEncode([]byte{1, 2, 3}) - e.MustEncode([]byte{4, 5, 6}) - buf.WriteByte(cborBdBreak) - - //---- - buf.WriteByte(cborBdIndefiniteString) - e.MustEncode("two-") - e.MustEncode("string-") - e.MustEncode("key") - buf.WriteByte(cborBdBreak) - - buf.WriteByte(cborBdIndefiniteString) - e.MustEncode([]byte("two-")) // encode as bytes, to check robustness of code - e.MustEncode([]byte("value")) - buf.WriteByte(cborBdBreak) - - //---- - buf.WriteByte(cborBdIndefiniteString) - e.MustEncode("three-") - e.MustEncode("list-") - e.MustEncode("key") - buf.WriteByte(cborBdBreak) - - buf.WriteByte(cborBdIndefiniteArray) - e.MustEncode(true) - e.MustEncode(false) - e.MustEncode(uint64(1)) - e.MustEncode(int64(-1)) - buf.WriteByte(cborBdBreak) - - buf.WriteByte(cborBdBreak) // close map - - NewDecoderBytes(buf.Bytes(), testCborH).MustDecode(&vv) - if err := deepEqual(v, vv); err != nil { - logT(t, "-------- Before and After marshal do not match: Error: %v", err) - logT(t, " ....... GOLDEN: (%T) %#v", v, v) - logT(t, " ....... DECODED: (%T) %#v", vv, vv) - failT(t) - } -} - -type testCborGolden struct { - Base64 string `codec:"cbor"` - Hex string `codec:"hex"` - Roundtrip bool `codec:"roundtrip"` - Decoded interface{} `codec:"decoded"` - Diagnostic string `codec:"diagnostic"` - Skip bool `codec:"skip"` -} - -// Some tests are skipped because they include numbers outside the range of int64/uint64 -func TestCborGoldens(t *testing.T) { - oldMapType := testCborH.MapType - defer func() { - testCborH.MapType = oldMapType - }() - testCborH.MapType = testMapStrIntfTyp - // decode test-cbor-goldens.json into a list of []*testCborGolden - // for each one, - // - decode hex into []byte bs - // - decode bs into interface{} v - // - compare both using deepequal - // - for any miss, record it - var gs []*testCborGolden - f, err := os.Open("test-cbor-goldens.json") - if err != nil { - logT(t, "error opening test-cbor-goldens.json: %v", err) - failT(t) - } - defer f.Close() - jh := new(JsonHandle) - jh.MapType = testMapStrIntfTyp - // d := NewDecoder(f, jh) - d := NewDecoder(bufio.NewReader(f), jh) - // err = d.Decode(&gs) - d.MustDecode(&gs) - if err != nil { - logT(t, "error json decoding test-cbor-goldens.json: %v", err) - failT(t) - } - - tagregex := regexp.MustCompile(`[\d]+\(.+?\)`) - hexregex := regexp.MustCompile(`h'([0-9a-fA-F]*)'`) - for i, g := range gs { - // fmt.Printf("%v, skip: %v, isTag: %v, %s\n", i, g.Skip, tagregex.MatchString(g.Diagnostic), g.Diagnostic) - // skip tags or simple or those with prefix, as we can't verify them. - if g.Skip || strings.HasPrefix(g.Diagnostic, "simple(") || tagregex.MatchString(g.Diagnostic) { - // fmt.Printf("%v: skipped\n", i) - logT(t, "[%v] skipping because skip=true OR unsupported simple value or Tag Value", i) - continue - } - // println("++++++++++++", i, "g.Diagnostic", g.Diagnostic) - if hexregex.MatchString(g.Diagnostic) { - // println(i, "g.Diagnostic matched hex") - if s2 := g.Diagnostic[2 : len(g.Diagnostic)-1]; s2 == "" { - g.Decoded = zeroByteSlice - } else if bs2, err2 := hex.DecodeString(s2); err2 == nil { - g.Decoded = bs2 - } - // fmt.Printf("%v: hex: %v\n", i, g.Decoded) - } - bs, err := hex.DecodeString(g.Hex) - if err != nil { - logT(t, "[%v] error hex decoding %s [%v]: %v", i, g.Hex, g.Hex, err) - failT(t) - } - var v interface{} - NewDecoderBytes(bs, testCborH).MustDecode(&v) - if _, ok := v.(RawExt); ok { - continue - } - // check the diagnostics to compare - switch g.Diagnostic { - case "Infinity": - b := math.IsInf(v.(float64), 1) - testCborError(t, i, math.Inf(1), v, nil, &b) - case "-Infinity": - b := math.IsInf(v.(float64), -1) - testCborError(t, i, math.Inf(-1), v, nil, &b) - case "NaN": - // println(i, "checking NaN") - b := math.IsNaN(v.(float64)) - testCborError(t, i, math.NaN(), v, nil, &b) - case "undefined": - b := v == nil - testCborError(t, i, nil, v, nil, &b) - default: - v0 := g.Decoded - // testCborCoerceJsonNumber(reflect.ValueOf(&v0)) - testCborError(t, i, v0, v, deepEqual(v0, v), nil) - } - } -} - -func testCborError(t *testing.T, i int, v0, v1 interface{}, err error, equal *bool) { - if err == nil && equal == nil { - // fmt.Printf("%v testCborError passed (err and equal nil)\n", i) - return - } - if err != nil { - logT(t, "[%v] deepEqual error: %v", i, err) - logT(t, " ....... GOLDEN: (%T) %#v", v0, v0) - logT(t, " ....... DECODED: (%T) %#v", v1, v1) - failT(t) - } - if equal != nil && !*equal { - logT(t, "[%v] values not equal", i) - logT(t, " ....... GOLDEN: (%T) %#v", v0, v0) - logT(t, " ....... DECODED: (%T) %#v", v1, v1) - failT(t) - } - // fmt.Printf("%v testCborError passed (checks passed)\n", i) -} - -func TestCborHalfFloat(t *testing.T) { - m := map[uint16]float64{ - // using examples from - // https://en.wikipedia.org/wiki/Half-precision_floating-point_format - 0x3c00: 1, - 0x3c01: 1 + math.Pow(2, -10), - 0xc000: -2, - 0x7bff: 65504, - 0x0400: math.Pow(2, -14), - 0x03ff: math.Pow(2, -14) - math.Pow(2, -24), - 0x0001: math.Pow(2, -24), - 0x0000: 0, - 0x8000: -0.0, - } - var ba [3]byte - ba[0] = cborBdFloat16 - var res float64 - for k, v := range m { - res = 0 - bigen.PutUint16(ba[1:], k) - testUnmarshalErr(&res, ba[:3], testCborH, t, "-") - if res == v { - logT(t, "equal floats: from %x %b, %v", k, k, v) - } else { - failT(t, "unequal floats: from %x %b, %v != %v", k, k, res, v) - } - } -} diff --git a/codec/codec_test.go b/codec/codec_test.go index af570ff6..70fca63c 100644 --- a/codec/codec_test.go +++ b/codec/codec_test.go @@ -25,6 +25,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/hashicorp/go-msgpack/v2/codec/internal" ) func init() { @@ -300,6 +302,163 @@ func (x *wrapBytesExt) UpdateExt(dest interface{}, v interface{}) { // ---- +// EncodeTime encodes a time.Time as a []byte, including +// information on the instant in time and UTC offset. +// +// Format Description +// +// A timestamp is composed of 3 components: +// +// - secs: signed integer representing seconds since unix epoch +// - nsces: unsigned integer representing fractional seconds as a +// nanosecond offset within secs, in the range 0 <= nsecs < 1e9 +// - tz: signed integer representing timezone offset in minutes east of UTC, +// and a dst (daylight savings time) flag +// +// When encoding a timestamp, the first byte is the descriptor, which +// defines which components are encoded and how many bytes are used to +// encode secs and nsecs components. *If secs/nsecs is 0 or tz is UTC, it +// is not encoded in the byte array explicitly*. +// +// Descriptor 8 bits are of the form `A B C DDD EE`: +// A: Is secs component encoded? 1 = true +// B: Is nsecs component encoded? 1 = true +// C: Is tz component encoded? 1 = true +// DDD: Number of extra bytes for secs (range 0-7). +// If A = 1, secs encoded in DDD+1 bytes. +// If A = 0, secs is not encoded, and is assumed to be 0. +// If A = 1, then we need at least 1 byte to encode secs. +// DDD says the number of extra bytes beyond that 1. +// E.g. if DDD=0, then secs is represented in 1 byte. +// if DDD=2, then secs is represented in 3 bytes. +// EE: Number of extra bytes for nsecs (range 0-3). +// If B = 1, nsecs encoded in EE+1 bytes (similar to secs/DDD above) +// +// Following the descriptor bytes, subsequent bytes are: +// +// secs component encoded in `DDD + 1` bytes (if A == 1) +// nsecs component encoded in `EE + 1` bytes (if B == 1) +// tz component encoded in 2 bytes (if C == 1) +// +// secs and nsecs components are integers encoded in a BigEndian +// 2-complement encoding format. +// +// tz component is encoded as 2 bytes (16 bits). Most significant bit 15 to +// Least significant bit 0 are described below: +// +// Timezone offset has a range of -12:00 to +14:00 (ie -720 to +840 minutes). +// Bit 15 = have\_dst: set to 1 if we set the dst flag. +// Bit 14 = dst\_on: set to 1 if dst is in effect at the time, or 0 if not. +// Bits 13..0 = timezone offset in minutes. It is a signed integer in Big Endian format. +func bincEncodeTime(t time.Time) []byte { + //t := rv.Interface().(time.Time) + tsecs, tnsecs := t.Unix(), t.Nanosecond() + var ( + bd byte + btmp [8]byte + bs [16]byte + i int = 1 + ) + l := t.Location() + if l == time.UTC { + l = nil + } + if tsecs != 0 { + bd = bd | 0x80 + bigen.PutUint64(btmp[:], uint64(tsecs)) + f := pruneSignExt(btmp[:], tsecs >= 0) + bd = bd | (byte(7-f) << 2) + copy(bs[i:], btmp[f:]) + i = i + (8 - f) + } + if tnsecs != 0 { + bd = bd | 0x40 + bigen.PutUint32(btmp[:4], uint32(tnsecs)) + f := pruneSignExt(btmp[:4], true) + bd = bd | byte(3-f) + copy(bs[i:], btmp[f:4]) + i = i + (4 - f) + } + if l != nil { + bd = bd | 0x20 + // Note that Go Libs do not give access to dst flag. + _, zoneOffset := t.Zone() + //zoneName, zoneOffset := t.Zone() + zoneOffset /= 60 + z := uint16(zoneOffset) + bigen.PutUint16(btmp[:2], z) + // clear dst flags + bs[i] = btmp[0] & 0x3f + bs[i+1] = btmp[1] + i = i + 2 + } + bs[0] = bd + return bs[0:i] +} + +// bincDecodeTime decodes a []byte into a time.Time. +func bincDecodeTime(bs []byte) (tt time.Time, err error) { + bd := bs[0] + var ( + tsec int64 + tnsec uint32 + tz uint16 + i byte = 1 + i2 byte + n byte + ) + if bd&(1<<7) != 0 { + var btmp [8]byte + n = ((bd >> 2) & 0x7) + 1 + i2 = i + n + copy(btmp[8-n:], bs[i:i2]) + //if first bit of bs[i] is set, then fill btmp[0..8-n] with 0xff (ie sign extend it) + if bs[i]&(1<<7) != 0 { + copy(btmp[0:8-n], bsAll0xff) + //for j,k := byte(0), 8-n; j < k; j++ { btmp[j] = 0xff } + } + i = i2 + tsec = int64(bigen.Uint64(btmp[:])) + } + if bd&(1<<6) != 0 { + var btmp [4]byte + n = (bd & 0x3) + 1 + i2 = i + n + copy(btmp[4-n:], bs[i:i2]) + i = i2 + tnsec = bigen.Uint32(btmp[:]) + } + if bd&(1<<5) == 0 { + tt = time.Unix(tsec, int64(tnsec)).UTC() + return + } + // In stdlib time.Parse, when a date is parsed without a zone name, it uses "" as zone name. + // However, we need name here, so it can be shown when time is printf.d. + // Zone name is in form: UTC-08:00. + // Note that Go Libs do not give access to dst flag, so we ignore dst bits + + i2 = i + 2 + tz = bigen.Uint16(bs[i:i2]) + // i = i2 + // sign extend sign bit into top 2 MSB (which were dst bits): + if tz&(1<<13) == 0 { // positive + tz = tz & 0x3fff //clear 2 MSBs: dst bits + } else { // negative + tz = tz | 0xc000 //set 2 MSBs: dst bits + } + tzint := int16(tz) + if tzint == 0 { + tt = time.Unix(tsec, int64(tnsec)).UTC() + } else { + // For Go Time, do not use a descriptive timezone. + // It's unnecessary, and makes it harder to do a reflect.DeepEqual. + // The Offset already tells what the offset should be, if not on UTC and unknown zone name. + // var zoneName = timeLocUTCName(tzint) + tt = time.Unix(tsec, int64(tnsec)).In(time.FixedZone("", int(tzint)*60)) + } + return +} + // timeExt is an extension handler for time.Time, that uses binc model for encoding/decoding time. // we used binc model, as that is the only custom time representation that we designed ourselves. type timeExt struct{} @@ -348,7 +507,7 @@ func checkErrT(t *testing.T, err error) { } func checkEqualT(t *testing.T, v1 interface{}, v2 interface{}, desc string) { - if err := deepEqual(v1, v2); err != nil { + if err := internal.DeepEqual(v1, v2); err != nil { failT(t, "Not Equal: %s: %v. v1: %v, v2: %v", desc, err, v1, v2) } } @@ -386,29 +545,9 @@ func testInit() { testMsgpackH.WriteExt = true - var tTimeExt timeExt var tBytesExt wrapBytesExt var tI64Ext wrapInt64Ext - // create legacy functions suitable for deprecated AddExt functionality, - // and use on some places for testSimpleH e.g. for time.Time and wrapInt64 - var ( - myExtEncFn = func(x BytesExt, rv reflect.Value) (bs []byte, err error) { - defer panicToErr(errDecoratorDef{}, &err) - bs = x.WriteExt(rv.Interface()) - return - } - myExtDecFn = func(x BytesExt, rv reflect.Value, bs []byte) (err error) { - defer panicToErr(errDecoratorDef{}, &err) - x.ReadExt(rv.Interface(), bs) - return - } - timeExtEncFn = func(rv reflect.Value) (bs []byte, err error) { return myExtEncFn(tTimeExt, rv) } - timeExtDecFn = func(rv reflect.Value, bs []byte) (err error) { return myExtDecFn(tTimeExt, rv, bs) } - wrapInt64ExtEncFn = func(rv reflect.Value) (bs []byte, err error) { return myExtEncFn(&tI64Ext, rv) } - wrapInt64ExtDecFn = func(rv reflect.Value, bs []byte) (err error) { return myExtDecFn(&tI64Ext, rv, bs) } - ) - chkErr := func(err error) { if err != nil { panic(err) @@ -417,27 +556,18 @@ func testInit() { // time.Time is a native type, so extensions will have no effect. // However, we add these here to ensure nothing happens. - chkErr(testSimpleH.AddExt(timeTyp, 1, timeExtEncFn, timeExtDecFn)) - // testBincH.SetBytesExt(timeTyp, 1, timeExt{}) // time is builtin for binc chkErr(testMsgpackH.SetBytesExt(timeTyp, 1, timeExt{})) - chkErr(testCborH.SetInterfaceExt(timeTyp, 1, &testUnixNanoTimeExt{})) // testJsonH.SetInterfaceExt(timeTyp, 1, &testUnixNanoTimeExt{}) // Now, add extensions for the type wrapInt64 and wrapBytes, // so we can execute the Encode/Decode Ext paths. - chkErr(testSimpleH.SetBytesExt(wrapBytesTyp, 32, &tBytesExt)) chkErr(testMsgpackH.SetBytesExt(wrapBytesTyp, 32, &tBytesExt)) - chkErr(testBincH.SetBytesExt(wrapBytesTyp, 32, &tBytesExt)) chkErr(testJsonH.SetInterfaceExt(wrapBytesTyp, 32, &tBytesExt)) - chkErr(testCborH.SetInterfaceExt(wrapBytesTyp, 32, &tBytesExt)) - chkErr(testSimpleH.AddExt(wrapInt64Typ, 16, wrapInt64ExtEncFn, wrapInt64ExtDecFn)) // chkErr(testSimpleH.SetBytesExt(wrapInt64Typ, 16, &tI64Ext)) chkErr(testMsgpackH.SetBytesExt(wrapInt64Typ, 16, &tI64Ext)) - chkErr(testBincH.SetBytesExt(wrapInt64Typ, 16, &tI64Ext)) chkErr(testJsonH.SetInterfaceExt(wrapInt64Typ, 16, &tI64Ext)) - chkErr(testCborH.SetInterfaceExt(wrapInt64Typ, 16, &tI64Ext)) // primitives MUST be an even number, so it can be used as a mapBySlice also. primitives := []interface{}{ @@ -601,7 +731,6 @@ func testVerifyVal(v interface{}, f testVerifyFlag, h Handle) (v2 interface{}) { // - all positive integers are unsigned 64-bit ints // - all floats are float64 _, isMsgp := h.(*MsgpackHandle) - _, isCbor := h.(*CborHandle) switch iv := v.(type) { case int8: v2 = testVerifyValInt(int64(iv), isMsgp) @@ -694,9 +823,6 @@ func testVerifyVal(v interface{}, f testVerifyFlag, h Handle) (v2 interface{}) { } case isMsgp: v2 = iv.UTC() - case isCbor: - // fmt.Printf("%%%% cbor verifier\n") - v2 = iv.UTC().Round(time.Microsecond) default: v2 = v } @@ -729,7 +855,7 @@ func testUnmarshalErr(v interface{}, data []byte, h Handle, t *testing.T, name s } func testDeepEqualErr(v1, v2 interface{}, t *testing.T, name string) { - if err := deepEqual(v1, v2); err == nil { + if err := internal.DeepEqual(v1, v2); err == nil { logT(t, "%s: values equal", name) } else { failT(t, "%s: values not equal: %v. 1: %v, 2: %v", name, err, v1, v2) @@ -801,7 +927,7 @@ func doTestCodecTableOne(t *testing.T, testNil bool, h Handle, continue } - if err = deepEqual(v0check, v1); err == nil { + if err = internal.DeepEqual(v0check, v1); err == nil { logT(t, "++++++++ Before and After marshal matched\n") } else { // logT(t, "-------- Before and After marshal do not match: Error: %v"+ @@ -921,13 +1047,13 @@ func testCodecMiscOne(t *testing.T, h Handle) { // log("m: %v, m2: %v, p: %v, p2: %v", m, m2, p, p2) checkEqualT(t, p, p2, "p=p2") checkEqualT(t, m, m2, "m=m2") - if err = deepEqual(p, p2); err == nil { + if err = internal.DeepEqual(p, p2); err == nil { logT(t, "p and p2 match") } else { logT(t, "Not Equal: %v. p: %v, p2: %v", err, p, p2) failT(t) } - if err = deepEqual(m, m2); err == nil { + if err = internal.DeepEqual(m, m2); err == nil { logT(t, "m and m2 match") } else { logT(t, "Not Equal: %v. m: %v, m2: %v", err, m, m2) @@ -1060,7 +1186,7 @@ func testCodecChan(t *testing.T, h Handle) { for j := range ch2 { sl2 = append(sl2, j) } - if err := deepEqual(sl1, sl2); err != nil { + if err := internal.DeepEqual(sl1, sl2); err != nil { logT(t, "FAIL: Not Match: %v; len: %v, %v", err, len(sl1), len(sl2)) failT(t) } @@ -1088,7 +1214,7 @@ func testCodecChan(t *testing.T, h Handle) { // logT(t, ">>>> from chan: is nil? %v, %v", j == nil, j) sl2 = append(sl2, j) } - if err := deepEqual(sl1, sl2); err != nil { + if err := internal.DeepEqual(sl1, sl2); err != nil { logT(t, "FAIL: Not Match: %v; len: %v, %v", err, len(sl1), len(sl2)) failT(t) } @@ -1114,7 +1240,7 @@ func testCodecChan(t *testing.T, h Handle) { for j := range ch2 { sl2 = append(sl2, j) } - if err := deepEqual(sl1, sl2); err != nil { + if err := internal.DeepEqual(sl1, sl2); err != nil { logT(t, "FAIL: Not Match: %v; len: %v, %v", err, len(sl1), len(sl2)) failT(t) } @@ -1140,7 +1266,7 @@ func testCodecChan(t *testing.T, h Handle) { for j := range ch2 { sl2 = append(sl2, j) } - if err := deepEqual(sl1, sl2); err != nil { + if err := internal.DeepEqual(sl1, sl2); err != nil { logT(t, "FAIL: Not Match: %v; len: %v, %v", err, len(sl1), len(sl2)) failT(t) } @@ -1294,7 +1420,7 @@ func doTestMapEncodeForCanonical(t *testing.T, name string, h Handle) { }, } var v2 map[stringUint64T]interface{} - var b1, b2, b3 []byte + var b1, b2 []byte // encode v1 into b1, decode b1 into v2, encode v2 into b2, and compare b1 and b2. // OR @@ -1303,10 +1429,6 @@ func doTestMapEncodeForCanonical(t *testing.T, name string, h Handle) { // where each key is encoded as an indefinite length string, which makes it not the same // order as the strings were lexicographically ordered before. - var cborIndef bool - if ch, ok := h.(*CborHandle); ok { - cborIndef = ch.IndefiniteLength - } bh := basicHandle(h) if !bh.Canonical { bh.Canonical = true @@ -1321,11 +1443,6 @@ func doTestMapEncodeForCanonical(t *testing.T, name string, h Handle) { e2 := NewEncoderBytes(&b2, h) e2.MustEncode(v2) var b1t, b2t = b1, b2 - if cborIndef { - e2 = NewEncoderBytes(&b3, h) - e2.MustEncode(v2) - b1t, b2t = b2, b3 - } if !bytes.Equal(b1t, b2t) { logT(t, "Unequal bytes: %v VS %v", b1t, b2t) @@ -1345,7 +1462,7 @@ func doTestStdEncIntf(t *testing.T, name string, h Handle) { e.MustEncode(a[0]) d := NewDecoderBytes(b, h) d.MustDecode(a[1]) - if err := deepEqual(a[0], a[1]); err == nil { + if err := internal.DeepEqual(a[0], a[1]); err == nil { logT(t, "++++ Objects match") } else { logT(t, "---- FAIL: Objects do not match: y1: %v, err: %v", a[1], err) @@ -1609,7 +1726,7 @@ func doTestPythonGenStreams(t *testing.T, name string, h Handle) { } //no need to indirect, because we pass a nil ptr, so we already have the value //if v1 != nil { v1 = reflect.Indirect(reflect.ValueOf(v1)).Interface() } - if err = deepEqual(v, v1); err == nil { + if err = internal.DeepEqual(v, v1); err == nil { logT(t, "++++++++ Objects match: %T, %v", v, v) } else { logT(t, "-------- FAIL: Objects do not match: %v. Source: %T. Decoded: %T", err, v, v1) @@ -1624,7 +1741,7 @@ func doTestPythonGenStreams(t *testing.T, name string, h Handle) { failT(t) continue } - if err = deepEqual(bsb, bss); err == nil { + if err = internal.DeepEqual(bsb, bss); err == nil { logT(t, "++++++++ Bytes match") } else { logT(t, "???????? FAIL: Bytes do not match. %v.", err) @@ -1722,7 +1839,6 @@ func doTestRawExt(t *testing.T, h Handle) { var b []byte var v RawExt // interface{} _, isJson := h.(*JsonHandle) - _, isCbor := h.(*CborHandle) bh := basicHandle(h) // isValuer := isJson || isCbor // _ = isValuer @@ -1739,8 +1855,6 @@ func doTestRawExt(t *testing.T, h Handle) { case isJson: r2.Tag = 0 r2.Data = nil - case isCbor: - r2.Data = nil default: r2.Value = nil } @@ -1935,11 +2049,6 @@ func doTestLargeContainerLen(t *testing.T, h Handle) { // to do this, we create a simple one-field struct, // use use flags to switch from symbols to non-symbols - hbinc, okbinc := h.(*BincHandle) - if okbinc { - oldAsSymbols := hbinc.AsSymbols - defer func() { hbinc.AsSymbols = oldAsSymbols }() - } var out []byte = make([]byte, 0, math.MaxUint16*3/2) var in []byte = make([]byte, math.MaxUint16*3/2) for i := range in { @@ -1960,9 +2069,6 @@ func doTestLargeContainerLen(t *testing.T, h Handle) { // fmt.Printf("testcontainerlen: large string: i: %v, |%s|\n", i, s1) m1[s1] = true - if okbinc { - hbinc.AsSymbols = 2 - } out = out[:0] e.ResetBytes(&out) e.MustEncode(m1) @@ -1970,18 +2076,6 @@ func doTestLargeContainerLen(t *testing.T, h Handle) { m2 = make(map[string]bool, 1) testUnmarshalErr(m2, out, h, t, "no-symbols") testDeepEqualErr(m1, m2, t, "no-symbols") - - if okbinc { - // now, do as symbols - hbinc.AsSymbols = 1 - out = out[:0] - e.ResetBytes(&out) - e.MustEncode(m1) - // bs, _ = testMarshalErr(m1, h, t, "-") - m2 = make(map[string]bool, 1) - testUnmarshalErr(m2, out, h, t, "symbols") - testDeepEqualErr(m1, m2, t, "symbols") - } } } @@ -2204,7 +2298,7 @@ func doTestDifferentMapOrSliceType(t *testing.T, name string, h Handle) { v2d = nil // v2d = reflect.New(bh.MapType).Elem().Interface() switch h.(type) { - case *MsgpackHandle, *BincHandle, *CborHandle: + case *MsgpackHandle: b = testMarshalErr(v2i, h, t, name) testUnmarshalErr(&v2d, b, h, t, name) testDeepEqualErr(v2d, goldMap[j], t, name) @@ -2481,7 +2575,7 @@ func TestJsonDecodeNonStringScalarInStringContext(t *testing.T) { var m map[string]string d := NewDecoderBytes([]byte(b), testJsonH) d.MustDecode(&m) - if err := deepEqual(golden, m); err == nil { + if err := internal.DeepEqual(golden, m); err == nil { logT(t, "++++ match: decoded: %#v", m) } else { logT(t, "---- mismatch: %v ==> golden: %#v, decoded: %#v", err, golden, m) @@ -2786,46 +2880,6 @@ func TestMsgpackDecodeMapAndExtSizeMismatch(t *testing.T) { // ---------- -func TestBincCodecsTable(t *testing.T) { - testCodecTableOne(t, testBincH) -} - -func TestBincCodecsMisc(t *testing.T) { - testCodecMiscOne(t, testBincH) -} - -func TestBincCodecsEmbeddedPointer(t *testing.T) { - testCodecEmbeddedPointer(t, testBincH) -} - -func TestBincStdEncIntf(t *testing.T) { - doTestStdEncIntf(t, "binc", testBincH) -} - -func TestBincMammoth(t *testing.T) { - testMammoth(t, "binc", testBincH) -} - -func TestSimpleCodecsTable(t *testing.T) { - testCodecTableOne(t, testSimpleH) -} - -func TestSimpleCodecsMisc(t *testing.T) { - testCodecMiscOne(t, testSimpleH) -} - -func TestSimpleCodecsEmbeddedPointer(t *testing.T) { - testCodecEmbeddedPointer(t, testSimpleH) -} - -func TestSimpleStdEncIntf(t *testing.T) { - doTestStdEncIntf(t, "simple", testSimpleH) -} - -func TestSimpleMammoth(t *testing.T) { - testMammoth(t, "simple", testSimpleH) -} - func TestMsgpackCodecsTable(t *testing.T) { testCodecTableOne(t, testMsgpackH) } @@ -2846,34 +2900,6 @@ func TestMsgpackMammoth(t *testing.T) { testMammoth(t, "msgpack", testMsgpackH) } -func TestCborCodecsTable(t *testing.T) { - testCodecTableOne(t, testCborH) -} - -func TestCborCodecsMisc(t *testing.T) { - testCodecMiscOne(t, testCborH) -} - -func TestCborCodecsEmbeddedPointer(t *testing.T) { - testCodecEmbeddedPointer(t, testCborH) -} - -func TestCborMapEncodeForCanonical(t *testing.T) { - doTestMapEncodeForCanonical(t, "cbor", testCborH) -} - -func TestCborCodecChan(t *testing.T) { - testCodecChan(t, testCborH) -} - -func TestCborStdEncIntf(t *testing.T) { - doTestStdEncIntf(t, "cbor", testCborH) -} - -func TestCborMammoth(t *testing.T) { - testMammoth(t, "cbor", testCborH) -} - func TestJsonCodecsTable(t *testing.T) { testCodecTableOne(t, testJsonH) } @@ -2902,52 +2928,22 @@ func TestJsonMammoth(t *testing.T) { func TestJsonRaw(t *testing.T) { doTestRawValue(t, "json", testJsonH) } -func TestBincRaw(t *testing.T) { - doTestRawValue(t, "binc", testBincH) -} func TestMsgpackRaw(t *testing.T) { doTestRawValue(t, "msgpack", testMsgpackH) } -func TestSimpleRaw(t *testing.T) { - doTestRawValue(t, "simple", testSimpleH) -} -func TestCborRaw(t *testing.T) { - doTestRawValue(t, "cbor", testCborH) -} // ----- ALL (framework based) ----- -func TestAllEncCircularRef(t *testing.T) { - doTestEncCircularRef(t, "cbor", testCborH) -} - -func TestAllAnonCycle(t *testing.T) { - doTestAnonCycle(t, "cbor", testCborH) -} - func TestAllErrWriter(t *testing.T) { - doTestErrWriter(t, "cbor", testCborH) doTestErrWriter(t, "json", testJsonH) } // ----- RPC ----- -func TestBincRpcGo(t *testing.T) { - testCodecRpcOne(t, GoRpc, testBincH, true, 0) -} - -func TestSimpleRpcGo(t *testing.T) { - testCodecRpcOne(t, GoRpc, testSimpleH, true, 0) -} - func TestMsgpackRpcGo(t *testing.T) { testCodecRpcOne(t, GoRpc, testMsgpackH, true, 0) } -func TestCborRpcGo(t *testing.T) { - testCodecRpcOne(t, GoRpc, testCborH, true, 0) -} - func TestJsonRpcGo(t *testing.T) { testCodecRpcOne(t, GoRpc, testJsonH, true, 0) } @@ -2956,138 +2952,58 @@ func TestMsgpackRpcSpec(t *testing.T) { testCodecRpcOne(t, MsgpackSpecRpc, testMsgpackH, true, 0) } -func TestBincUnderlyingType(t *testing.T) { - testCodecUnderlyingType(t, testBincH) -} - func TestJsonSwallowAndZero(t *testing.T) { doTestSwallowAndZero(t, testJsonH) } -func TestCborSwallowAndZero(t *testing.T) { - doTestSwallowAndZero(t, testCborH) -} - func TestMsgpackSwallowAndZero(t *testing.T) { doTestSwallowAndZero(t, testMsgpackH) } -func TestBincSwallowAndZero(t *testing.T) { - doTestSwallowAndZero(t, testBincH) -} - -func TestSimpleSwallowAndZero(t *testing.T) { - doTestSwallowAndZero(t, testSimpleH) -} - func TestJsonRawExt(t *testing.T) { doTestRawExt(t, testJsonH) } -func TestCborRawExt(t *testing.T) { - doTestRawExt(t, testCborH) -} - func TestMsgpackRawExt(t *testing.T) { doTestRawExt(t, testMsgpackH) } -func TestBincRawExt(t *testing.T) { - doTestRawExt(t, testBincH) -} - -func TestSimpleRawExt(t *testing.T) { - doTestRawExt(t, testSimpleH) -} - func TestJsonMapStructKey(t *testing.T) { doTestMapStructKey(t, testJsonH) } -func TestCborMapStructKey(t *testing.T) { - doTestMapStructKey(t, testCborH) -} - func TestMsgpackMapStructKey(t *testing.T) { doTestMapStructKey(t, testMsgpackH) } -func TestBincMapStructKey(t *testing.T) { - doTestMapStructKey(t, testBincH) -} - -func TestSimpleMapStructKey(t *testing.T) { - doTestMapStructKey(t, testSimpleH) -} - func TestJsonDecodeNilMapValue(t *testing.T) { doTestDecodeNilMapValue(t, testJsonH) } -func TestCborDecodeNilMapValue(t *testing.T) { - doTestDecodeNilMapValue(t, testCborH) -} - func TestMsgpackDecodeNilMapValue(t *testing.T) { doTestDecodeNilMapValue(t, testMsgpackH) } -func TestBincDecodeNilMapValue(t *testing.T) { - doTestDecodeNilMapValue(t, testBincH) -} - -func TestSimpleDecodeNilMapValue(t *testing.T) { - doTestDecodeNilMapValue(t, testSimpleH) -} - func TestJsonEmbeddedFieldPrecedence(t *testing.T) { doTestEmbeddedFieldPrecedence(t, testJsonH) } -func TestCborEmbeddedFieldPrecedence(t *testing.T) { - doTestEmbeddedFieldPrecedence(t, testCborH) -} - func TestMsgpackEmbeddedFieldPrecedence(t *testing.T) { doTestEmbeddedFieldPrecedence(t, testMsgpackH) } -func TestBincEmbeddedFieldPrecedence(t *testing.T) { - doTestEmbeddedFieldPrecedence(t, testBincH) -} - -func TestSimpleEmbeddedFieldPrecedence(t *testing.T) { - doTestEmbeddedFieldPrecedence(t, testSimpleH) -} - func TestJsonLargeContainerLen(t *testing.T) { doTestLargeContainerLen(t, testJsonH) } -func TestCborLargeContainerLen(t *testing.T) { - doTestLargeContainerLen(t, testCborH) -} - func TestMsgpackLargeContainerLen(t *testing.T) { doTestLargeContainerLen(t, testMsgpackH) } -func TestBincLargeContainerLen(t *testing.T) { - doTestLargeContainerLen(t, testBincH) -} - -func TestSimpleLargeContainerLen(t *testing.T) { - doTestLargeContainerLen(t, testSimpleH) -} - func TestJsonMammothMapsAndSlices(t *testing.T) { doTestMammothMapsAndSlices(t, testJsonH) } -func TestCborMammothMapsAndSlices(t *testing.T) { - doTestMammothMapsAndSlices(t, testCborH) -} - func TestMsgpackMammothMapsAndSlices(t *testing.T) { old1 := testMsgpackH.WriteExt defer func() { testMsgpackH.WriteExt = old1 }() @@ -3096,174 +3012,70 @@ func TestMsgpackMammothMapsAndSlices(t *testing.T) { doTestMammothMapsAndSlices(t, testMsgpackH) } -func TestBincMammothMapsAndSlices(t *testing.T) { - doTestMammothMapsAndSlices(t, testBincH) -} - -func TestSimpleMammothMapsAndSlices(t *testing.T) { - doTestMammothMapsAndSlices(t, testSimpleH) -} - func TestJsonTime(t *testing.T) { testTime(t, "json", testJsonH) } -func TestCborTime(t *testing.T) { - testTime(t, "cbor", testCborH) -} - func TestMsgpackTime(t *testing.T) { testTime(t, "msgpack", testMsgpackH) } -func TestBincTime(t *testing.T) { - testTime(t, "binc", testBincH) -} - -func TestSimpleTime(t *testing.T) { - testTime(t, "simple", testSimpleH) -} - func TestJsonUintToInt(t *testing.T) { testUintToInt(t, "json", testJsonH) } -func TestCborUintToInt(t *testing.T) { - testUintToInt(t, "cbor", testCborH) -} - func TestMsgpackUintToInt(t *testing.T) { testUintToInt(t, "msgpack", testMsgpackH) } -func TestBincUintToInt(t *testing.T) { - testUintToInt(t, "binc", testBincH) -} - -func TestSimpleUintToInt(t *testing.T) { - testUintToInt(t, "simple", testSimpleH) -} - func TestJsonDifferentMapOrSliceType(t *testing.T) { doTestDifferentMapOrSliceType(t, "json", testJsonH) } -func TestCborDifferentMapOrSliceType(t *testing.T) { - doTestDifferentMapOrSliceType(t, "cbor", testCborH) -} - func TestMsgpackDifferentMapOrSliceType(t *testing.T) { doTestDifferentMapOrSliceType(t, "msgpack", testMsgpackH) } -func TestBincDifferentMapOrSliceType(t *testing.T) { - doTestDifferentMapOrSliceType(t, "binc", testBincH) -} - -func TestSimpleDifferentMapOrSliceType(t *testing.T) { - doTestDifferentMapOrSliceType(t, "simple", testSimpleH) -} - func TestJsonScalars(t *testing.T) { doTestScalars(t, "json", testJsonH) } -func TestCborScalars(t *testing.T) { - doTestScalars(t, "cbor", testCborH) -} - func TestMsgpackScalars(t *testing.T) { doTestScalars(t, "msgpack", testMsgpackH) } -func TestBincScalars(t *testing.T) { - doTestScalars(t, "binc", testBincH) -} - -func TestSimpleScalars(t *testing.T) { - doTestScalars(t, "simple", testSimpleH) -} - func TestJsonOmitempty(t *testing.T) { doTestOmitempty(t, "json", testJsonH) } -func TestCborOmitempty(t *testing.T) { - doTestOmitempty(t, "cbor", testCborH) -} - func TestMsgpackOmitempty(t *testing.T) { doTestOmitempty(t, "msgpack", testMsgpackH) } -func TestBincOmitempty(t *testing.T) { - doTestOmitempty(t, "binc", testBincH) -} - -func TestSimpleOmitempty(t *testing.T) { - doTestOmitempty(t, "simple", testSimpleH) -} - func TestJsonIntfMapping(t *testing.T) { doTestIntfMapping(t, "json", testJsonH) } -func TestCborIntfMapping(t *testing.T) { - doTestIntfMapping(t, "cbor", testCborH) -} - func TestMsgpackIntfMapping(t *testing.T) { doTestIntfMapping(t, "msgpack", testMsgpackH) } -func TestBincIntfMapping(t *testing.T) { - doTestIntfMapping(t, "binc", testBincH) -} - -func TestSimpleIntfMapping(t *testing.T) { - doTestIntfMapping(t, "simple", testSimpleH) -} - func TestJsonMissingFields(t *testing.T) { doTestMissingFields(t, "json", testJsonH) } -func TestCborMissingFields(t *testing.T) { - doTestMissingFields(t, "cbor", testCborH) -} - func TestMsgpackMissingFields(t *testing.T) { doTestMissingFields(t, "msgpack", testMsgpackH) } -func TestBincMissingFields(t *testing.T) { - doTestMissingFields(t, "binc", testBincH) -} - -func TestSimpleMissingFields(t *testing.T) { - doTestMissingFields(t, "simple", testSimpleH) -} - func TestJsonMaxDepth(t *testing.T) { doTestMaxDepth(t, "json", testJsonH) } -func TestCborMaxDepth(t *testing.T) { - doTestMaxDepth(t, "cbor", testCborH) -} - func TestMsgpackMaxDepth(t *testing.T) { doTestMaxDepth(t, "msgpack", testMsgpackH) } -func TestBincMaxDepth(t *testing.T) { - doTestMaxDepth(t, "binc", testBincH) -} - -func TestSimpleMaxDepth(t *testing.T) { - doTestMaxDepth(t, "simple", testSimpleH) -} - func TestMultipleEncDec(t *testing.T) { doTestMultipleEncDec(t, "json", testJsonH) } diff --git a/codec/codecgen/README.md b/codec/codecgen/README.md index 854b64bf..f6b5ad34 100644 --- a/codec/codecgen/README.md +++ b/codec/codecgen/README.md @@ -9,7 +9,7 @@ Using codecgen is very straightforward. **Download and install the tool** -`go get -u github.com/ugorji/go/codec/codecgen` +`go get -u github.com/hashicorp/go-msgpack/v2/codec/codecgen` **Run the tool on your files** @@ -20,7 +20,7 @@ The command line format is: ```sh % codecgen -? Usage of codecgen: - -c="github.com/ugorji/go/codec": codec path + -c="github.com/hashicorp/go-msgpack/v2/codec": codec path -o="": out file -r=".*": regex for type name to match -nr="": regex for type name to exclude diff --git a/codec/decode.go b/codec/decode.go index 381f2bab..e0fb62df 100644 --- a/codec/decode.go +++ b/codec/decode.go @@ -1277,8 +1277,8 @@ func (d *Decoder) kInterfaceNaked(f *codecFnInfo) (rvn reflect.Value) { var v2 []interface{} d.decode(&v2) rvn = reflect.ValueOf(&v2).Elem() - if reflectArrayOfSupported && d.stid == 0 && d.h.PreferArrayOverSlice { - rvn2 := reflect.New(reflectArrayOf(rvn.Len(), intfTyp)).Elem() + if d.stid == 0 && d.h.PreferArrayOverSlice { + rvn2 := reflect.New(reflect.ArrayOf(rvn.Len(), intfTyp)).Elem() reflect.Copy(rvn2, rvn) rvn = rvn2 } diff --git a/codec/doc.go b/codec/doc.go index b828068f..deac0001 100644 --- a/codec/doc.go +++ b/codec/doc.go @@ -2,38 +2,26 @@ // Use of this source code is governed by a MIT license found in the LICENSE file. /* -Package codec provides a -High Performance, Feature-Rich Idiomatic Go 1.4+ codec/encoding library -for binc, msgpack, cbor, json. +Package codec provides a High Performance, Feature-Rich Idiomatic +codec/encoding library for msgpack, json. Supported Serialization formats are: - msgpack: https://github.com/msgpack/msgpack - - binc: http://github.com/ugorji/binc - - cbor: http://cbor.io http://tools.ietf.org/html/rfc7049 - json: http://json.org http://tools.ietf.org/html/rfc7159 - - simple: -This package will carefully use 'package unsafe' for performance reasons in specific places. -You can build without unsafe use by passing the safe or appengine tag -i.e. 'go install -tags=safe ...'. Note that unsafe is only supported for the last 4 -go releases e.g. current go release is go 1.12, so we support unsafe use only from -go 1.9+ . This is because supporting unsafe requires knowledge of implementation details. +For detailed usage information, read the primer at +http://ugorji.net/blog/go-codec-primer . -For detailed usage information, read the primer at http://ugorji.net/blog/go-codec-primer . - -The idiomatic Go support is as seen in other encoding packages in -the standard library (ie json, xml, gob, etc). +The idiomatic Go support is as seen in other encoding packages in the +standard library (ie json, xml, gob, etc). Rich Feature Set includes: - Simple but extremely powerful and feature-rich API - - Support for go1.4 and above, while selectively using newer APIs for later releases - Excellent code coverage ( > 90% ) - Very High Performance. Our extensive benchmarks show us outperforming Gob, Json, Bson, etc by 2-4X. - - Careful selected use of 'unsafe' for targeted performance gains. - 100% mode exists where 'unsafe' is not used at all. - Lock-free (sans mutex) concurrency for scaling to 100's of cores - In-place updates during decode, with option to zero value in maps and slices prior to decode - Coerce types where appropriate @@ -59,9 +47,9 @@ Rich Feature Set includes: - Comprehensive support for anonymous fields - Fast (no-reflection) encoding/decoding of common maps and slices - Code-generation for faster performance. - - Support binary (e.g. messagepack, cbor) and text (e.g. json) formats + - Support binary (e.g. messagepack) and text (e.g. json) formats - Support indefinite-length formats to enable true streaming - (for formats which support it e.g. json, cbor) + (for formats which support it e.g. json) - Support canonical encoding, where a value is ALWAYS encoded as same sequence of bytes. This mostly applies to maps, where iteration order is non-deterministic. - NIL in data stream decoded as zero value @@ -73,33 +61,37 @@ Rich Feature Set includes: - Drop-in replacement for encoding/json. `json:` key in struct tag supported. - Provides a RPC Server and Client Codec for net/rpc communication protocol. - Handle unique idiosyncrasies of codecs e.g. - - For messagepack, configure how ambiguities in handling raw bytes are resolved - - For messagepack, provide rpc server/client codec to support - msgpack-rpc protocol defined at: - https://github.com/msgpack-rpc/msgpack-rpc/blob/master/spec.md + - For messagepack, configure how ambiguities in handling raw bytes are resolved + - For messagepack, provide rpc server/client codec to support + msgpack-rpc protocol defined at: + https://github.com/msgpack-rpc/msgpack-rpc/blob/master/spec.md + -# Extension Support +## Extension Support -Users can register a function to handle the encoding or decoding of -their custom types. +Users can register a function to handle the encoding or decoding of their +custom types. There are no restrictions on what the custom type can be. Some examples: - type BisSet []int - type BitSet64 uint64 - type UUID string - type MyStructWithUnexportedFields struct { a int; b bool; c []int; } - type GifImage struct { ... } +```go + type BisSet []int + type BitSet64 uint64 + type UUID string + type MyStructWithUnexportedFields struct { a int; b bool; c []int; } + type GifImage struct { ... } +``` -As an illustration, MyStructWithUnexportedFields would normally be -encoded as an empty map because it has no exported fields, while UUID -would be encoded as a string. However, with extension support, you can -encode any of these however you like. +As an illustration, MyStructWithUnexportedFields would normally be encoded +as an empty map because it has no exported fields, while UUID would be +encoded as a string. However, with extension support, you can encode any of +these however you like. -# Custom Encoding and Decoding -This package maintains symmetry in the encoding and decoding halfs. -We determine how to encode or decode by walking this decision tree +## Custom Encoding and Decoding + +This package maintains symmetry in the encoding and decoding halfs. We +determine how to encode or decode by walking this decision tree - is type a codec.Selfer? - is there an extension registered for the type? @@ -112,19 +104,21 @@ This symmetry is important to reduce chances of issues happening because the encoding and decoding sides are out of sync e.g. decoded via very specific encoding.TextUnmarshaler but encoded via kind-specific generalized mode. -Consequently, if a type only defines one-half of the symmetry -(e.g. it implements UnmarshalJSON() but not MarshalJSON() ), -then that type doesn't satisfy the check and we will continue walking down the -decision tree. +Consequently, if a type only defines one-half of the symmetry (e.g. it +implements UnmarshalJSON() but not MarshalJSON() ), then that type doesn't +satisfy the check and we will continue walking down the decision tree. + -# RPC +## RPC -RPC Client and Server Codecs are implemented, so the codecs can be used -with the standard net/rpc package. +RPC Client and Server Codecs are implemented, so the codecs can be used with +the standard net/rpc package. -# Usage -The Handle is SAFE for concurrent READ, but NOT SAFE for concurrent modification. +## Usage + +The Handle is SAFE for concurrent READ, but NOT SAFE for concurrent +modification. The Encoder and Decoder are NOT safe for concurrent use. @@ -140,85 +134,100 @@ Consequently, the usage model is basically: Sample usage model: - // create and configure Handle - var ( - bh codec.BincHandle - mh codec.MsgpackHandle - ch codec.CborHandle - ) - - mh.MapType = reflect.TypeOf(map[string]interface{}(nil)) - - // configure extensions - // e.g. for msgpack, define functions and enable Time support for tag 1 - // mh.SetExt(reflect.TypeOf(time.Time{}), 1, myExt) - - // create and use decoder/encoder - var ( - r io.Reader - w io.Writer - b []byte - h = &bh // or mh to use msgpack - ) - - dec = codec.NewDecoder(r, h) - dec = codec.NewDecoderBytes(b, h) - err = dec.Decode(&v) - - enc = codec.NewEncoder(w, h) - enc = codec.NewEncoderBytes(&b, h) - err = enc.Encode(v) - - //RPC Server - go func() { - for { - conn, err := listener.Accept() - rpcCodec := codec.GoRpc.ServerCodec(conn, h) - //OR rpcCodec := codec.MsgpackSpecRpc.ServerCodec(conn, h) - rpc.ServeCodec(rpcCodec) - } - }() - - //RPC Communication (client side) - conn, err = net.Dial("tcp", "localhost:5555") - rpcCodec := codec.GoRpc.ClientCodec(conn, h) - //OR rpcCodec := codec.MsgpackSpecRpc.ClientCodec(conn, h) - client := rpc.NewClientWithCodec(rpcCodec) - -# Running Tests +```go + // create and configure Handle + var ( + mh codec.MsgpackHandle + ) + + mh.MapType = reflect.TypeOf(map[string]interface{}(nil)) + + // configure extensions + // e.g. for msgpack, define functions and enable Time support for tag 1 + mh.SetExt(reflect.TypeOf(time.Time{}), 1, myExt) + + // create and use decoder/encoder + var ( + r io.Reader + w io.Writer + b []byte + h = &mh + ) + + dec = codec.NewDecoder(r, h) + dec = codec.NewDecoderBytes(b, h) + err = dec.Decode(&v) + + enc = codec.NewEncoder(w, h) + enc = codec.NewEncoderBytes(&b, h) + err = enc.Encode(v) + + //RPC Server + go func() { + for { + conn, err := listener.Accept() + rpcCodec := codec.GoRpc.ServerCodec(conn, h) + //OR rpcCodec := codec.MsgpackSpecRpc.ServerCodec(conn, h) + rpc.ServeCodec(rpcCodec) + } + }() + + //RPC Communication (client side) + conn, err = net.Dial("tcp", "localhost:5555") + rpcCodec := codec.GoRpc.ClientCodec(conn, h) + //OR rpcCodec := codec.MsgpackSpecRpc.ClientCodec(conn, h) + client := rpc.NewClientWithCodec(rpcCodec) +``` + + +## Running Tests To run tests, use the following: - go test +``` + go test +``` To run the full suite of tests, use the following: - go test -tags alltests -run Suite +``` + go test -tags alltests -run Suite +``` You can run the tag 'safe' to run tests or build in safe mode. e.g. - go test -tags safe -run Json - go test -tags "alltests safe" -run Suite +``` + go test -tags safe -run Json + go test -tags "alltests safe" -run Suite +``` + +## Running Benchmarks + +``` + cd codec/bench + ./bench.sh -d + ./bench.sh -c + ./bench.sh -s + go test -bench . -benchmem -benchtime 1s +``` -Running Benchmarks +Please see http://github.com/hashicorp/go-codec-bench . - cd bench - go test -bench . -benchmem -benchtime 1s -Please see http://github.com/ugorji/go-codec-bench . +## Caveats -# Caveats +Struct fields matching the following are ignored during encoding and +decoding -Struct fields matching the following are ignored during encoding and decoding - struct tag value set to - - func, complex numbers, unsafe pointers - unexported and not embedded - unexported and embedded and not struct kind - - unexported and embedded pointers (from go1.10) + - unexported and embedded pointers Every other field in a struct will be encoded/decoded. -Embedded fields are encoded as if they exist in the top-level struct, -with some caveats. See Encode documentation. +Embedded fields are encoded as if they exist in the top-level struct, with +some caveats. See Encode documentation. */ package codec diff --git a/codec/gen.go b/codec/gen.go index 73c6242a..efaff02e 100644 --- a/codec/gen.go +++ b/codec/gen.go @@ -1740,10 +1740,7 @@ func (x *genV) MethodNamePfx(prefix string, prim bool) string { // We strip it here. func genImportPath(t reflect.Type) (s string) { s = t.PkgPath() - if genCheckVendor { - // HACK: always handle vendoring. It should be typically on in go 1.6, 1.7 - s = genStripVendor(s) - } + s = genStripVendor(s) return } diff --git a/codec/goversion_arrayof_gte_go15.go b/codec/goversion_arrayof_gte_go15.go deleted file mode 100644 index 7978a777..00000000 --- a/codec/goversion_arrayof_gte_go15.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -//go:build go1.5 -// +build go1.5 - -package codec - -import "reflect" - -const reflectArrayOfSupported = true - -func reflectArrayOf(count int, elem reflect.Type) reflect.Type { - return reflect.ArrayOf(count, elem) -} diff --git a/codec/goversion_arrayof_lt_go15.go b/codec/goversion_arrayof_lt_go15.go deleted file mode 100644 index bcff2783..00000000 --- a/codec/goversion_arrayof_lt_go15.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -//go:build !go1.5 -// +build !go1.5 - -package codec - -import "reflect" - -const reflectArrayOfSupported = false - -func reflectArrayOf(count int, elem reflect.Type) reflect.Type { - panic("codec: reflect.ArrayOf unsupported in this go version") -} diff --git a/codec/goversion_makemap_gte_go19.go b/codec/goversion_makemap_gte_go19.go deleted file mode 100644 index 49323557..00000000 --- a/codec/goversion_makemap_gte_go19.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -//go:build go1.9 -// +build go1.9 - -package codec - -import "reflect" - -func makeMapReflect(t reflect.Type, size int) reflect.Value { - if size < 0 { - return reflect.MakeMapWithSize(t, 4) - } - return reflect.MakeMapWithSize(t, size) -} diff --git a/codec/goversion_makemap_lt_go19.go b/codec/goversion_makemap_lt_go19.go deleted file mode 100644 index 4602a536..00000000 --- a/codec/goversion_makemap_lt_go19.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -//go:build !go1.9 -// +build !go1.9 - -package codec - -import "reflect" - -func makeMapReflect(t reflect.Type, size int) reflect.Value { - return reflect.MakeMap(t) -} diff --git a/codec/goversion_unexportedembeddedptr_gte_go110.go b/codec/goversion_unexportedembeddedptr_gte_go110.go deleted file mode 100644 index ec4d3fae..00000000 --- a/codec/goversion_unexportedembeddedptr_gte_go110.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -//go:build go1.10 -// +build go1.10 - -package codec - -const allowSetUnexportedEmbeddedPtr = false diff --git a/codec/goversion_unexportedembeddedptr_lt_go110.go b/codec/goversion_unexportedembeddedptr_lt_go110.go deleted file mode 100644 index 49f24cf1..00000000 --- a/codec/goversion_unexportedembeddedptr_lt_go110.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -//go:build !go1.10 -// +build !go1.10 - -package codec - -const allowSetUnexportedEmbeddedPtr = true diff --git a/codec/goversion_unsupported_lt_go14.go b/codec/goversion_unsupported_lt_go14.go deleted file mode 100644 index d3524be3..00000000 --- a/codec/goversion_unsupported_lt_go14.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -//go:build !go1.4 -// +build !go1.4 - -package codec - -// This codec package will only work for go1.4 and above. -// This is for the following reasons: -// - go 1.4 was released in 2014 -// - go runtime is written fully in go -// - interface only holds pointers -// - reflect.Value is stabilized as 3 words - -func init() { - panic("codec: go 1.3 and below are not supported") -} diff --git a/codec/goversion_vendor_eq_go15.go b/codec/goversion_vendor_eq_go15.go deleted file mode 100644 index a7103b02..00000000 --- a/codec/goversion_vendor_eq_go15.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -//go:build go1.5 && !go1.6 -// +build go1.5,!go1.6 - -package codec - -import "os" - -var genCheckVendor = os.Getenv("GO15VENDOREXPERIMENT") == "1" diff --git a/codec/goversion_vendor_eq_go16.go b/codec/goversion_vendor_eq_go16.go deleted file mode 100644 index 28d332c4..00000000 --- a/codec/goversion_vendor_eq_go16.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -//go:build go1.6 && !go1.7 -// +build go1.6,!go1.7 - -package codec - -import "os" - -var genCheckVendor = os.Getenv("GO15VENDOREXPERIMENT") != "0" diff --git a/codec/goversion_vendor_gte_go17.go b/codec/goversion_vendor_gte_go17.go deleted file mode 100644 index 50513afb..00000000 --- a/codec/goversion_vendor_gte_go17.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -//go:build go1.7 -// +build go1.7 - -package codec - -const genCheckVendor = true diff --git a/codec/goversion_vendor_lt_go15.go b/codec/goversion_vendor_lt_go15.go deleted file mode 100644 index 5a3079d2..00000000 --- a/codec/goversion_vendor_lt_go15.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -//go:build !go1.5 -// +build !go1.5 - -package codec - -var genCheckVendor = false diff --git a/codec/helper.go b/codec/helper.go index 78aba686..9c0ed16a 100644 --- a/codec/helper.go +++ b/codec/helper.go @@ -1712,7 +1712,7 @@ LOOP: // Also, from go1.10, ignore pointers to unexported struct types // because unmarshal cannot assign a new struct to an unexported field. // See https://golang.org/issue/21357 - if (isUnexported && !isStruct) || (!allowSetUnexportedEmbeddedPtr && isUnexported && isPtr) { + if (isUnexported && !isStruct) || (isUnexported && isPtr) { continue } doInline := stag == "" @@ -2917,3 +2917,10 @@ func (e *Encoder) kUint64(f *codecFnInfo, rv reflect.Value) { func (e *Encoder) kUintptr(f *codecFnInfo, rv reflect.Value) { e.e.EncodeUint(rv.Uint()) } + +func makeMapReflect(t reflect.Type, size int) reflect.Value { + if size < 0 { + return reflect.MakeMapWithSize(t, 4) + } + return reflect.MakeMapWithSize(t, size) +} diff --git a/codec/helper_internal.go b/codec/helper_internal.go index 0cbd665e..4f5eb843 100644 --- a/codec/helper_internal.go +++ b/codec/helper_internal.go @@ -18,38 +18,6 @@ func pruneSignExt(v []byte, pos bool) (n int) { return } -// validate that this function is correct ... -// culled from OGRE (Object-Oriented Graphics Rendering Engine) -// function: halfToFloatI (http://stderr.org/doc/ogre-doc/api/OgreBitwise_8h-source.html) -func halfFloatToFloatBits(yy uint16) (d uint32) { - y := uint32(yy) - s := (y >> 15) & 0x01 - e := (y >> 10) & 0x1f - m := y & 0x03ff - - if e == 0 { - if m == 0 { // plu or minus 0 - return s << 31 - } - // Denormalized number -- renormalize it - for (m & 0x00000400) == 0 { - m <<= 1 - e -= 1 - } - e += 1 - const zz uint32 = 0x0400 - m &= ^zz - } else if e == 31 { - if m == 0 { // Inf - return (s << 31) | 0x7f800000 - } - return (s << 31) | 0x7f800000 | (m << 13) // NaN - } - e = e + (127 - 15) - m = m << 13 - return (s << 31) | (e << 23) | m -} - // GrowCap will return a new capacity for a slice, given the following: // - oldCap: current capacity // - unit: in-memory size of an element diff --git a/codec/helper_test.go b/codec/internal/test_helper.go similarity index 73% rename from codec/helper_test.go rename to codec/internal/test_helper.go index fe2879ec..f53a4d0f 100644 --- a/codec/helper_test.go +++ b/codec/internal/test_helper.go @@ -1,7 +1,7 @@ // Copyright (c) 2012-2015 Ugorji Nwoke. All rights reserved. // Use of this source code is governed by a MIT license found in the LICENSE file. -package codec +package internal // All non-std package dependencies related to testing live in this file, // so porting to different environment is easy (just update functions). @@ -13,25 +13,25 @@ import ( // --- these functions are used by both benchmarks and tests -var errDeepEqualNotMatch = errors.New("Not Match") +var ErrDeepEqualNotMatch = errors.New("not Match") -func deepEqual(v1, v2 interface{}) (err error) { +func DeepEqual(v1, v2 interface{}) (err error) { if !reflect.DeepEqual(v1, v2) { - err = errDeepEqualNotMatch + err = ErrDeepEqualNotMatch } return } -func approxDataSize(rv reflect.Value) (sum int) { +func ApproxDataSize(rv reflect.Value) (sum int) { switch rk := rv.Kind(); rk { case reflect.Invalid: case reflect.Ptr, reflect.Interface: sum += int(rv.Type().Size()) - sum += approxDataSize(rv.Elem()) + sum += ApproxDataSize(rv.Elem()) case reflect.Slice: sum += int(rv.Type().Size()) for j := 0; j < rv.Len(); j++ { - sum += approxDataSize(rv.Index(j)) + sum += ApproxDataSize(rv.Index(j)) } case reflect.String: sum += int(rv.Type().Size()) @@ -39,14 +39,14 @@ func approxDataSize(rv reflect.Value) (sum int) { case reflect.Map: sum += int(rv.Type().Size()) for _, mk := range rv.MapKeys() { - sum += approxDataSize(mk) - sum += approxDataSize(rv.MapIndex(mk)) + sum += ApproxDataSize(mk) + sum += ApproxDataSize(rv.MapIndex(mk)) } case reflect.Struct: //struct size already includes the full data size. //sum += int(rv.Type().Size()) for j := 0; j < rv.NumField(); j++ { - sum += approxDataSize(rv.Field(j)) + sum += ApproxDataSize(rv.Field(j)) } default: //pure value types diff --git a/codec/py_test.go b/codec/py_test.go index 0310d88e..a53d1fb1 100644 --- a/codec/py_test.go +++ b/codec/py_test.go @@ -18,10 +18,6 @@ func TestMsgpackPythonGenStreams(t *testing.T) { doTestPythonGenStreams(t, "msgpack", testMsgpackH) } -func TestCborPythonGenStreams(t *testing.T) { - doTestPythonGenStreams(t, "cbor", testCborH) -} - func TestMsgpackRpcSpecGoClientToPythonSvc(t *testing.T) { doTestMsgpackRpcSpecGoClientToPythonSvc(t) } diff --git a/codec/shared_test.go b/codec/shared_test.go index 17893dd2..0a453a6e 100644 --- a/codec/shared_test.go +++ b/codec/shared_test.go @@ -78,9 +78,6 @@ func (x ioWriterWrapper) Write(p []byte) (n int, err error) { var ( // testNoopH = NoopHandle(8) testMsgpackH = &MsgpackHandle{} - testBincH = &BincHandle{} - testSimpleH = &SimpleHandle{} - testCborH = &CborHandle{} testJsonH = &JsonHandle{} testHandles []Handle @@ -139,12 +136,9 @@ func init() { testHEDs = make([]testHED, 0, 32) testHandles = append(testHandles, // testNoopH, - testMsgpackH, testBincH, testSimpleH, testCborH, testJsonH) + testMsgpackH, testJsonH) // set ExplicitRelease on each handle testMsgpackH.ExplicitRelease = true - testBincH.ExplicitRelease = true - testSimpleH.ExplicitRelease = true - testCborH.ExplicitRelease = true testJsonH.ExplicitRelease = true testInitFlags() diff --git a/codec/simple.go b/codec/simple.go deleted file mode 100644 index a3257c1a..00000000 --- a/codec/simple.go +++ /dev/null @@ -1,666 +0,0 @@ -// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -package codec - -import ( - "math" - "reflect" - "time" -) - -const ( - _ uint8 = iota - simpleVdNil = 1 - simpleVdFalse = 2 - simpleVdTrue = 3 - simpleVdFloat32 = 4 - simpleVdFloat64 = 5 - - // each lasts for 4 (ie n, n+1, n+2, n+3) - simpleVdPosInt = 8 - simpleVdNegInt = 12 - - simpleVdTime = 24 - - // containers: each lasts for 4 (ie n, n+1, n+2, ... n+7) - simpleVdString = 216 - simpleVdByteArray = 224 - simpleVdArray = 232 - simpleVdMap = 240 - simpleVdExt = 248 -) - -type simpleEncDriver struct { - noBuiltInTypes - // encNoSeparator - e *Encoder - h *SimpleHandle - w *encWriterSwitch - b [8]byte - // c containerState - encDriverTrackContainerWriter - // encDriverNoopContainerWriter - _ [3]uint64 // padding -} - -func (e *simpleEncDriver) EncodeNil() { - e.w.writen1(simpleVdNil) -} - -func (e *simpleEncDriver) EncodeBool(b bool) { - if e.h.EncZeroValuesAsNil && e.c != containerMapKey && !b { - e.EncodeNil() - return - } - if b { - e.w.writen1(simpleVdTrue) - } else { - e.w.writen1(simpleVdFalse) - } -} - -func (e *simpleEncDriver) EncodeFloat32(f float32) { - if e.h.EncZeroValuesAsNil && e.c != containerMapKey && f == 0.0 { - e.EncodeNil() - return - } - e.w.writen1(simpleVdFloat32) - bigenHelper{e.b[:4], e.w}.writeUint32(math.Float32bits(f)) -} - -func (e *simpleEncDriver) EncodeFloat64(f float64) { - if e.h.EncZeroValuesAsNil && e.c != containerMapKey && f == 0.0 { - e.EncodeNil() - return - } - e.w.writen1(simpleVdFloat64) - bigenHelper{e.b[:8], e.w}.writeUint64(math.Float64bits(f)) -} - -func (e *simpleEncDriver) EncodeInt(v int64) { - if v < 0 { - e.encUint(uint64(-v), simpleVdNegInt) - } else { - e.encUint(uint64(v), simpleVdPosInt) - } -} - -func (e *simpleEncDriver) EncodeUint(v uint64) { - e.encUint(v, simpleVdPosInt) -} - -func (e *simpleEncDriver) encUint(v uint64, bd uint8) { - if e.h.EncZeroValuesAsNil && e.c != containerMapKey && v == 0 { - e.EncodeNil() - return - } - if v <= math.MaxUint8 { - e.w.writen2(bd, uint8(v)) - } else if v <= math.MaxUint16 { - e.w.writen1(bd + 1) - bigenHelper{e.b[:2], e.w}.writeUint16(uint16(v)) - } else if v <= math.MaxUint32 { - e.w.writen1(bd + 2) - bigenHelper{e.b[:4], e.w}.writeUint32(uint32(v)) - } else { // if v <= math.MaxUint64 { - e.w.writen1(bd + 3) - bigenHelper{e.b[:8], e.w}.writeUint64(v) - } -} - -func (e *simpleEncDriver) encLen(bd byte, length int) { - if length == 0 { - e.w.writen1(bd) - } else if length <= math.MaxUint8 { - e.w.writen1(bd + 1) - e.w.writen1(uint8(length)) - } else if length <= math.MaxUint16 { - e.w.writen1(bd + 2) - bigenHelper{e.b[:2], e.w}.writeUint16(uint16(length)) - } else if int64(length) <= math.MaxUint32 { - e.w.writen1(bd + 3) - bigenHelper{e.b[:4], e.w}.writeUint32(uint32(length)) - } else { - e.w.writen1(bd + 4) - bigenHelper{e.b[:8], e.w}.writeUint64(uint64(length)) - } -} - -func (e *simpleEncDriver) EncodeExt(rv interface{}, xtag uint64, ext Ext, _ *Encoder) { - bs := ext.WriteExt(rv) - if bs == nil { - e.EncodeNil() - return - } - e.encodeExtPreamble(uint8(xtag), len(bs)) - e.w.writeb(bs) -} - -func (e *simpleEncDriver) EncodeRawExt(re *RawExt, _ *Encoder) { - e.encodeExtPreamble(uint8(re.Tag), len(re.Data)) - e.w.writeb(re.Data) -} - -func (e *simpleEncDriver) encodeExtPreamble(xtag byte, length int) { - e.encLen(simpleVdExt, length) - e.w.writen1(xtag) -} - -func (e *simpleEncDriver) WriteArrayStart(length int) { - e.c = containerArrayStart - e.encLen(simpleVdArray, length) -} - -func (e *simpleEncDriver) WriteMapStart(length int) { - e.c = containerMapStart - e.encLen(simpleVdMap, length) -} - -// func (e *simpleEncDriver) EncodeSymbol(v string) { -// e.EncodeStringEnc(cUTF8, v) -// } - -func (e *simpleEncDriver) EncodeStringEnc(c charEncoding, v string) { - if false && e.h.EncZeroValuesAsNil && e.c != containerMapKey && v == "" { - e.EncodeNil() - return - } - e.encLen(simpleVdString, len(v)) - e.w.writestr(v) -} - -func (e *simpleEncDriver) EncodeString(c charEncoding, v string) { - e.EncodeStringEnc(c, v) -} - -func (e *simpleEncDriver) EncodeStringBytes(c charEncoding, v []byte) { - e.EncodeStringBytesRaw(v) -} - -func (e *simpleEncDriver) EncodeStringBytesRaw(v []byte) { - // if e.h.EncZeroValuesAsNil && e.c != containerMapKey && v == nil { - if v == nil { - e.EncodeNil() - return - } - e.encLen(simpleVdByteArray, len(v)) - e.w.writeb(v) -} - -func (e *simpleEncDriver) EncodeTime(t time.Time) { - // if e.h.EncZeroValuesAsNil && e.c != containerMapKey && t.IsZero() { - if t.IsZero() { - e.EncodeNil() - return - } - v, err := t.MarshalBinary() - if err != nil { - e.e.errorv(err) - return - } - // time.Time marshalbinary takes about 14 bytes. - e.w.writen2(simpleVdTime, uint8(len(v))) - e.w.writeb(v) -} - -//------------------------------------ - -type simpleDecDriver struct { - d *Decoder - h *SimpleHandle - r *decReaderSwitch - bdRead bool - bd byte - br bool // a bytes reader? - c containerState - // b [scratchByteArrayLen]byte - noBuiltInTypes - // noStreamingCodec - decDriverNoopContainerReader - // _ [3]uint64 // padding -} - -func (d *simpleDecDriver) readNextBd() { - d.bd = d.r.readn1() - d.bdRead = true -} - -func (d *simpleDecDriver) uncacheRead() { - if d.bdRead { - d.r.unreadn1() - d.bdRead = false - } -} - -func (d *simpleDecDriver) ContainerType() (vt valueType) { - if !d.bdRead { - d.readNextBd() - } - switch d.bd { - case simpleVdNil: - return valueTypeNil - case simpleVdByteArray, simpleVdByteArray + 1, - simpleVdByteArray + 2, simpleVdByteArray + 3, simpleVdByteArray + 4: - return valueTypeBytes - case simpleVdString, simpleVdString + 1, - simpleVdString + 2, simpleVdString + 3, simpleVdString + 4: - return valueTypeString - case simpleVdArray, simpleVdArray + 1, - simpleVdArray + 2, simpleVdArray + 3, simpleVdArray + 4: - return valueTypeArray - case simpleVdMap, simpleVdMap + 1, - simpleVdMap + 2, simpleVdMap + 3, simpleVdMap + 4: - return valueTypeMap - // case simpleVdTime: - // return valueTypeTime - } - // else { - // d.d.errorf("isContainerType: unsupported parameter: %v", vt) - // } - return valueTypeUnset -} - -func (d *simpleDecDriver) TryDecodeAsNil() bool { - if !d.bdRead { - d.readNextBd() - } - if d.bd == simpleVdNil { - d.bdRead = false - return true - } - return false -} - -func (d *simpleDecDriver) decCheckInteger() (ui uint64, neg bool) { - if !d.bdRead { - d.readNextBd() - } - switch d.bd { - case simpleVdPosInt: - ui = uint64(d.r.readn1()) - case simpleVdPosInt + 1: - ui = uint64(bigen.Uint16(d.r.readx(2))) - case simpleVdPosInt + 2: - ui = uint64(bigen.Uint32(d.r.readx(4))) - case simpleVdPosInt + 3: - ui = uint64(bigen.Uint64(d.r.readx(8))) - case simpleVdNegInt: - ui = uint64(d.r.readn1()) - neg = true - case simpleVdNegInt + 1: - ui = uint64(bigen.Uint16(d.r.readx(2))) - neg = true - case simpleVdNegInt + 2: - ui = uint64(bigen.Uint32(d.r.readx(4))) - neg = true - case simpleVdNegInt + 3: - ui = uint64(bigen.Uint64(d.r.readx(8))) - neg = true - default: - d.d.errorf("integer only valid from pos/neg integer1..8. Invalid descriptor: %v", d.bd) - return - } - // don't do this check, because callers may only want the unsigned value. - // if ui > math.MaxInt64 { - // d.d.errorf("decIntAny: Integer out of range for signed int64: %v", ui) - // return - // } - return -} - -func (d *simpleDecDriver) DecodeInt64() (i int64) { - ui, neg := d.decCheckInteger() - i = chkOvf.SignedIntV(ui) - if neg { - i = -i - } - d.bdRead = false - return -} - -func (d *simpleDecDriver) DecodeUint64() (ui uint64) { - ui, neg := d.decCheckInteger() - if neg { - d.d.errorf("assigning negative signed value to unsigned type") - return - } - d.bdRead = false - return -} - -func (d *simpleDecDriver) DecodeFloat64() (f float64) { - if !d.bdRead { - d.readNextBd() - } - if d.bd == simpleVdFloat32 { - f = float64(math.Float32frombits(bigen.Uint32(d.r.readx(4)))) - } else if d.bd == simpleVdFloat64 { - f = math.Float64frombits(bigen.Uint64(d.r.readx(8))) - } else { - if d.bd >= simpleVdPosInt && d.bd <= simpleVdNegInt+3 { - f = float64(d.DecodeInt64()) - } else { - d.d.errorf("float only valid from float32/64: Invalid descriptor: %v", d.bd) - return - } - } - d.bdRead = false - return -} - -// bool can be decoded from bool only (single byte). -func (d *simpleDecDriver) DecodeBool() (b bool) { - if !d.bdRead { - d.readNextBd() - } - if d.bd == simpleVdTrue { - b = true - } else if d.bd == simpleVdFalse { - } else { - d.d.errorf("cannot decode bool - %s: %x", msgBadDesc, d.bd) - return - } - d.bdRead = false - return -} - -func (d *simpleDecDriver) ReadMapStart() (length int) { - if !d.bdRead { - d.readNextBd() - } - d.bdRead = false - d.c = containerMapStart - return d.decLen() -} - -func (d *simpleDecDriver) ReadArrayStart() (length int) { - if !d.bdRead { - d.readNextBd() - } - d.bdRead = false - d.c = containerArrayStart - return d.decLen() -} - -func (d *simpleDecDriver) ReadArrayElem() { - d.c = containerArrayElem -} - -func (d *simpleDecDriver) ReadArrayEnd() { - d.c = containerArrayEnd -} - -func (d *simpleDecDriver) ReadMapElemKey() { - d.c = containerMapKey -} - -func (d *simpleDecDriver) ReadMapElemValue() { - d.c = containerMapValue -} - -func (d *simpleDecDriver) ReadMapEnd() { - d.c = containerMapEnd -} - -func (d *simpleDecDriver) decLen() int { - switch d.bd % 8 { - case 0: - return 0 - case 1: - return int(d.r.readn1()) - case 2: - return int(bigen.Uint16(d.r.readx(2))) - case 3: - ui := uint64(bigen.Uint32(d.r.readx(4))) - if chkOvf.Uint(ui, intBitsize) { - d.d.errorf("overflow integer: %v", ui) - return 0 - } - return int(ui) - case 4: - ui := bigen.Uint64(d.r.readx(8)) - if chkOvf.Uint(ui, intBitsize) { - d.d.errorf("overflow integer: %v", ui) - return 0 - } - return int(ui) - } - d.d.errorf("cannot read length: bd%%8 must be in range 0..4. Got: %d", d.bd%8) - return -1 -} - -func (d *simpleDecDriver) DecodeString() (s string) { - return string(d.DecodeBytes(d.d.b[:], true)) -} - -func (d *simpleDecDriver) DecodeStringAsBytes() (s []byte) { - return d.DecodeBytes(d.d.b[:], true) -} - -func (d *simpleDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) { - if !d.bdRead { - d.readNextBd() - } - if d.bd == simpleVdNil { - d.bdRead = false - return - } - // check if an "array" of uint8's (see ContainerType for how to infer if an array) - if d.bd >= simpleVdArray && d.bd <= simpleVdMap+4 { - if len(bs) == 0 && zerocopy { - bs = d.d.b[:] - } - bsOut, _ = fastpathTV.DecSliceUint8V(bs, true, d.d) - return - } - - clen := d.decLen() - d.bdRead = false - if zerocopy { - if d.br { - return d.r.readx(uint(clen)) - } else if len(bs) == 0 { - bs = d.d.b[:] - } - } - return decByteSlice(d.r, clen, d.d.h.MaxInitLen, bs) -} - -func (d *simpleDecDriver) DecodeTime() (t time.Time) { - if !d.bdRead { - d.readNextBd() - } - if d.bd == simpleVdNil { - d.bdRead = false - return - } - if d.bd != simpleVdTime { - d.d.errorf("invalid descriptor for time.Time - expect 0x%x, received 0x%x", simpleVdTime, d.bd) - return - } - d.bdRead = false - clen := int(d.r.readn1()) - b := d.r.readx(uint(clen)) - if err := (&t).UnmarshalBinary(b); err != nil { - d.d.errorv(err) - } - return -} - -func (d *simpleDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) { - if xtag > 0xff { - d.d.errorf("ext: tag must be <= 0xff; got: %v", xtag) - return - } - realxtag1, xbs := d.decodeExtV(ext != nil, uint8(xtag)) - realxtag = uint64(realxtag1) - if ext == nil { - re := rv.(*RawExt) - re.Tag = realxtag - re.Data = detachZeroCopyBytes(d.br, re.Data, xbs) - } else { - ext.ReadExt(rv, xbs) - } - return -} - -func (d *simpleDecDriver) decodeExtV(verifyTag bool, tag byte) (xtag byte, xbs []byte) { - if !d.bdRead { - d.readNextBd() - } - switch d.bd { - case simpleVdExt, simpleVdExt + 1, simpleVdExt + 2, simpleVdExt + 3, simpleVdExt + 4: - l := d.decLen() - xtag = d.r.readn1() - if verifyTag && xtag != tag { - d.d.errorf("wrong extension tag. Got %b. Expecting: %v", xtag, tag) - return - } - if d.br { - xbs = d.r.readx(uint(l)) - } else { - xbs = decByteSlice(d.r, l, d.d.h.MaxInitLen, d.d.b[:]) - } - case simpleVdByteArray, simpleVdByteArray + 1, - simpleVdByteArray + 2, simpleVdByteArray + 3, simpleVdByteArray + 4: - xbs = d.DecodeBytes(nil, true) - default: - d.d.errorf("ext - %s - expecting extensions/bytearray, got: 0x%x", msgBadDesc, d.bd) - return - } - d.bdRead = false - return -} - -func (d *simpleDecDriver) DecodeNaked() { - if !d.bdRead { - d.readNextBd() - } - - n := d.d.naked() - var decodeFurther bool - - switch d.bd { - case simpleVdNil: - n.v = valueTypeNil - case simpleVdFalse: - n.v = valueTypeBool - n.b = false - case simpleVdTrue: - n.v = valueTypeBool - n.b = true - case simpleVdPosInt, simpleVdPosInt + 1, simpleVdPosInt + 2, simpleVdPosInt + 3: - if d.h.SignedInteger { - n.v = valueTypeInt - n.i = d.DecodeInt64() - } else { - n.v = valueTypeUint - n.u = d.DecodeUint64() - } - case simpleVdNegInt, simpleVdNegInt + 1, simpleVdNegInt + 2, simpleVdNegInt + 3: - n.v = valueTypeInt - n.i = d.DecodeInt64() - case simpleVdFloat32: - n.v = valueTypeFloat - n.f = d.DecodeFloat64() - case simpleVdFloat64: - n.v = valueTypeFloat - n.f = d.DecodeFloat64() - case simpleVdTime: - n.v = valueTypeTime - n.t = d.DecodeTime() - case simpleVdString, simpleVdString + 1, - simpleVdString + 2, simpleVdString + 3, simpleVdString + 4: - n.v = valueTypeString - n.s = d.DecodeString() - case simpleVdByteArray, simpleVdByteArray + 1, - simpleVdByteArray + 2, simpleVdByteArray + 3, simpleVdByteArray + 4: - decNakedReadRawBytes(d, d.d, n, d.h.RawToString) - case simpleVdExt, simpleVdExt + 1, simpleVdExt + 2, simpleVdExt + 3, simpleVdExt + 4: - n.v = valueTypeExt - l := d.decLen() - n.u = uint64(d.r.readn1()) - if d.br { - n.l = d.r.readx(uint(l)) - } else { - n.l = decByteSlice(d.r, l, d.d.h.MaxInitLen, d.d.b[:]) - } - case simpleVdArray, simpleVdArray + 1, simpleVdArray + 2, - simpleVdArray + 3, simpleVdArray + 4: - n.v = valueTypeArray - decodeFurther = true - case simpleVdMap, simpleVdMap + 1, simpleVdMap + 2, simpleVdMap + 3, simpleVdMap + 4: - n.v = valueTypeMap - decodeFurther = true - default: - d.d.errorf("cannot infer value - %s 0x%x", msgBadDesc, d.bd) - } - - if !decodeFurther { - d.bdRead = false - } -} - -//------------------------------------ - -// SimpleHandle is a Handle for a very simple encoding format. -// -// simple is a simplistic codec similar to binc, but not as compact. -// - Encoding of a value is always preceded by the descriptor byte (bd) -// - True, false, nil are encoded fully in 1 byte (the descriptor) -// - Integers (intXXX, uintXXX) are encoded in 1, 2, 4 or 8 bytes (plus a descriptor byte). -// There are positive (uintXXX and intXXX >= 0) and negative (intXXX < 0) integers. -// - Floats are encoded in 4 or 8 bytes (plus a descriptor byte) -// - Length of containers (strings, bytes, array, map, extensions) -// are encoded in 0, 1, 2, 4 or 8 bytes. -// Zero-length containers have no length encoded. -// For others, the number of bytes is given by pow(2, bd%3) -// - maps are encoded as [bd] [length] [[key][value]]... -// - arrays are encoded as [bd] [length] [value]... -// - extensions are encoded as [bd] [length] [tag] [byte]... -// - strings/bytearrays are encoded as [bd] [length] [byte]... -// - time.Time are encoded as [bd] [length] [byte]... -// -// The full spec will be published soon. -type SimpleHandle struct { - BasicHandle - binaryEncodingType - noElemSeparators - // EncZeroValuesAsNil says to encode zero values for numbers, bool, string, etc as nil - EncZeroValuesAsNil bool - - // _ [1]uint64 // padding -} - -// Name returns the name of the handle: simple -func (h *SimpleHandle) Name() string { return "simple" } - -// SetBytesExt sets an extension -func (h *SimpleHandle) SetBytesExt(rt reflect.Type, tag uint64, ext BytesExt) (err error) { - return h.SetExt(rt, tag, &extWrapper{ext, interfaceExtFailer{}}) -} - -func (h *SimpleHandle) hasElemSeparators() bool { return true } // as it implements Write(Map|Array)XXX - -func (h *SimpleHandle) newEncDriver(e *Encoder) encDriver { - return &simpleEncDriver{e: e, w: e.w, h: h} -} - -func (h *SimpleHandle) newDecDriver(d *Decoder) decDriver { - return &simpleDecDriver{d: d, h: h, r: d.r, br: d.bytes} -} - -func (e *simpleEncDriver) reset() { - e.c = 0 - e.w = e.e.w -} - -func (d *simpleDecDriver) reset() { - d.c = 0 - d.r, d.br = d.d.r, d.d.bytes - d.bd, d.bdRead = 0, false -} - -var _ decDriver = (*simpleDecDriver)(nil) -var _ encDriver = (*simpleEncDriver)(nil) diff --git a/codec/test-cbor-goldens.json b/codec/test-cbor-goldens.json deleted file mode 100644 index 90285867..00000000 --- a/codec/test-cbor-goldens.json +++ /dev/null @@ -1,639 +0,0 @@ -[ - { - "cbor": "AA==", - "hex": "00", - "roundtrip": true, - "decoded": 0 - }, - { - "cbor": "AQ==", - "hex": "01", - "roundtrip": true, - "decoded": 1 - }, - { - "cbor": "Cg==", - "hex": "0a", - "roundtrip": true, - "decoded": 10 - }, - { - "cbor": "Fw==", - "hex": "17", - "roundtrip": true, - "decoded": 23 - }, - { - "cbor": "GBg=", - "hex": "1818", - "roundtrip": true, - "decoded": 24 - }, - { - "cbor": "GBk=", - "hex": "1819", - "roundtrip": true, - "decoded": 25 - }, - { - "cbor": "GGQ=", - "hex": "1864", - "roundtrip": true, - "decoded": 100 - }, - { - "cbor": "GQPo", - "hex": "1903e8", - "roundtrip": true, - "decoded": 1000 - }, - { - "cbor": "GgAPQkA=", - "hex": "1a000f4240", - "roundtrip": true, - "decoded": 1000000 - }, - { - "cbor": "GwAAAOjUpRAA", - "hex": "1b000000e8d4a51000", - "roundtrip": true, - "decoded": 1000000000000 - }, - { - "cbor": "G///////////", - "hex": "1bffffffffffffffff", - "roundtrip": true, - "decoded": 18446744073709551615 - }, - { - "cbor": "wkkBAAAAAAAAAAA=", - "hex": "c249010000000000000000", - "roundtrip": true, - "decoded": 18446744073709551616 - }, - { - "cbor": "O///////////", - "hex": "3bffffffffffffffff", - "roundtrip": true, - "decoded": -18446744073709551616, - "skip": true - }, - { - "cbor": "w0kBAAAAAAAAAAA=", - "hex": "c349010000000000000000", - "roundtrip": true, - "decoded": -18446744073709551617 - }, - { - "cbor": "IA==", - "hex": "20", - "roundtrip": true, - "decoded": -1 - }, - { - "cbor": "KQ==", - "hex": "29", - "roundtrip": true, - "decoded": -10 - }, - { - "cbor": "OGM=", - "hex": "3863", - "roundtrip": true, - "decoded": -100 - }, - { - "cbor": "OQPn", - "hex": "3903e7", - "roundtrip": true, - "decoded": -1000 - }, - { - "cbor": "+QAA", - "hex": "f90000", - "roundtrip": true, - "decoded": 0.0 - }, - { - "cbor": "+YAA", - "hex": "f98000", - "roundtrip": true, - "decoded": -0.0 - }, - { - "cbor": "+TwA", - "hex": "f93c00", - "roundtrip": true, - "decoded": 1.0 - }, - { - "cbor": "+z/xmZmZmZma", - "hex": "fb3ff199999999999a", - "roundtrip": true, - "decoded": 1.1 - }, - { - "cbor": "+T4A", - "hex": "f93e00", - "roundtrip": true, - "decoded": 1.5 - }, - { - "cbor": "+Xv/", - "hex": "f97bff", - "roundtrip": true, - "decoded": 65504.0 - }, - { - "cbor": "+kfDUAA=", - "hex": "fa47c35000", - "roundtrip": true, - "decoded": 100000.0 - }, - { - "cbor": "+n9///8=", - "hex": "fa7f7fffff", - "roundtrip": true, - "decoded": 3.4028234663852886e+38 - }, - { - "cbor": "+3435DyIAHWc", - "hex": "fb7e37e43c8800759c", - "roundtrip": true, - "decoded": 1.0e+300 - }, - { - "cbor": "+QAB", - "hex": "f90001", - "roundtrip": true, - "decoded": 5.960464477539063e-08 - }, - { - "cbor": "+QQA", - "hex": "f90400", - "roundtrip": true, - "decoded": 6.103515625e-05 - }, - { - "cbor": "+cQA", - "hex": "f9c400", - "roundtrip": true, - "decoded": -4.0 - }, - { - "cbor": "+8AQZmZmZmZm", - "hex": "fbc010666666666666", - "roundtrip": true, - "decoded": -4.1 - }, - { - "cbor": "+XwA", - "hex": "f97c00", - "roundtrip": true, - "diagnostic": "Infinity" - }, - { - "cbor": "+X4A", - "hex": "f97e00", - "roundtrip": true, - "diagnostic": "NaN" - }, - { - "cbor": "+fwA", - "hex": "f9fc00", - "roundtrip": true, - "diagnostic": "-Infinity" - }, - { - "cbor": "+n+AAAA=", - "hex": "fa7f800000", - "roundtrip": false, - "diagnostic": "Infinity" - }, - { - "cbor": "+n/AAAA=", - "hex": "fa7fc00000", - "roundtrip": false, - "diagnostic": "NaN" - }, - { - "cbor": "+v+AAAA=", - "hex": "faff800000", - "roundtrip": false, - "diagnostic": "-Infinity" - }, - { - "cbor": "+3/wAAAAAAAA", - "hex": "fb7ff0000000000000", - "roundtrip": false, - "diagnostic": "Infinity" - }, - { - "cbor": "+3/4AAAAAAAA", - "hex": "fb7ff8000000000000", - "roundtrip": false, - "diagnostic": "NaN" - }, - { - "cbor": "+//wAAAAAAAA", - "hex": "fbfff0000000000000", - "roundtrip": false, - "diagnostic": "-Infinity" - }, - { - "cbor": "9A==", - "hex": "f4", - "roundtrip": true, - "decoded": false - }, - { - "cbor": "9Q==", - "hex": "f5", - "roundtrip": true, - "decoded": true - }, - { - "cbor": "9g==", - "hex": "f6", - "roundtrip": true, - "decoded": null - }, - { - "cbor": "9w==", - "hex": "f7", - "roundtrip": true, - "diagnostic": "undefined" - }, - { - "cbor": "8A==", - "hex": "f0", - "roundtrip": true, - "diagnostic": "simple(16)" - }, - { - "cbor": "+Bg=", - "hex": "f818", - "roundtrip": true, - "diagnostic": "simple(24)" - }, - { - "cbor": "+P8=", - "hex": "f8ff", - "roundtrip": true, - "diagnostic": "simple(255)" - }, - { - "cbor": "wHQyMDEzLTAzLTIxVDIwOjA0OjAwWg==", - "hex": "c074323031332d30332d32315432303a30343a30305a", - "roundtrip": true, - "diagnostic": "0(\"2013-03-21T20:04:00Z\")" - }, - { - "cbor": "wRpRS2ew", - "hex": "c11a514b67b0", - "roundtrip": true, - "diagnostic": "1(1363896240)" - }, - { - "cbor": "wftB1FLZ7CAAAA==", - "hex": "c1fb41d452d9ec200000", - "roundtrip": true, - "diagnostic": "1(1363896240.5)" - }, - { - "cbor": "10QBAgME", - "hex": "d74401020304", - "roundtrip": true, - "diagnostic": "23(h'01020304')" - }, - { - "cbor": "2BhFZElFVEY=", - "hex": "d818456449455446", - "roundtrip": true, - "diagnostic": "24(h'6449455446')" - }, - { - "cbor": "2CB2aHR0cDovL3d3dy5leGFtcGxlLmNvbQ==", - "hex": "d82076687474703a2f2f7777772e6578616d706c652e636f6d", - "roundtrip": true, - "diagnostic": "32(\"http://www.example.com\")" - }, - { - "cbor": "QA==", - "hex": "40", - "roundtrip": true, - "diagnostic": "h''" - }, - { - "cbor": "RAECAwQ=", - "hex": "4401020304", - "roundtrip": true, - "diagnostic": "h'01020304'" - }, - { - "cbor": "YA==", - "hex": "60", - "roundtrip": true, - "decoded": "" - }, - { - "cbor": "YWE=", - "hex": "6161", - "roundtrip": true, - "decoded": "a" - }, - { - "cbor": "ZElFVEY=", - "hex": "6449455446", - "roundtrip": true, - "decoded": "IETF" - }, - { - "cbor": "YiJc", - "hex": "62225c", - "roundtrip": true, - "decoded": "\"\\" - }, - { - "cbor": "YsO8", - "hex": "62c3bc", - "roundtrip": true, - "decoded": "ü" - }, - { - "cbor": "Y+awtA==", - "hex": "63e6b0b4", - "roundtrip": true, - "decoded": "水" - }, - { - "cbor": "ZPCQhZE=", - "hex": "64f0908591", - "roundtrip": true, - "decoded": "𐅑" - }, - { - "cbor": "gA==", - "hex": "80", - "roundtrip": true, - "decoded": [ - - ] - }, - { - "cbor": "gwECAw==", - "hex": "83010203", - "roundtrip": true, - "decoded": [ - 1, - 2, - 3 - ] - }, - { - "cbor": "gwGCAgOCBAU=", - "hex": "8301820203820405", - "roundtrip": true, - "decoded": [ - 1, - [ - 2, - 3 - ], - [ - 4, - 5 - ] - ] - }, - { - "cbor": "mBkBAgMEBQYHCAkKCwwNDg8QERITFBUWFxgYGBk=", - "hex": "98190102030405060708090a0b0c0d0e0f101112131415161718181819", - "roundtrip": true, - "decoded": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25 - ] - }, - { - "cbor": "oA==", - "hex": "a0", - "roundtrip": true, - "decoded": { - } - }, - { - "cbor": "ogECAwQ=", - "hex": "a201020304", - "roundtrip": true, - "skip": true, - "diagnostic": "{1: 2, 3: 4}" - }, - { - "cbor": "omFhAWFiggID", - "hex": "a26161016162820203", - "roundtrip": true, - "decoded": { - "a": 1, - "b": [ - 2, - 3 - ] - } - }, - { - "cbor": "gmFhoWFiYWM=", - "hex": "826161a161626163", - "roundtrip": true, - "decoded": [ - "a", - { - "b": "c" - } - ] - }, - { - "cbor": "pWFhYUFhYmFCYWNhQ2FkYURhZWFF", - "hex": "a56161614161626142616361436164614461656145", - "roundtrip": true, - "decoded": { - "a": "A", - "b": "B", - "c": "C", - "d": "D", - "e": "E" - } - }, - { - "cbor": "X0IBAkMDBAX/", - "hex": "5f42010243030405ff", - "roundtrip": false, - "skip": true, - "diagnostic": "(_ h'0102', h'030405')" - }, - { - "cbor": "f2VzdHJlYWRtaW5n/w==", - "hex": "7f657374726561646d696e67ff", - "roundtrip": false, - "decoded": "streaming" - }, - { - "cbor": "n/8=", - "hex": "9fff", - "roundtrip": false, - "decoded": [ - - ] - }, - { - "cbor": "nwGCAgOfBAX//w==", - "hex": "9f018202039f0405ffff", - "roundtrip": false, - "decoded": [ - 1, - [ - 2, - 3 - ], - [ - 4, - 5 - ] - ] - }, - { - "cbor": "nwGCAgOCBAX/", - "hex": "9f01820203820405ff", - "roundtrip": false, - "decoded": [ - 1, - [ - 2, - 3 - ], - [ - 4, - 5 - ] - ] - }, - { - "cbor": "gwGCAgOfBAX/", - "hex": "83018202039f0405ff", - "roundtrip": false, - "decoded": [ - 1, - [ - 2, - 3 - ], - [ - 4, - 5 - ] - ] - }, - { - "cbor": "gwGfAgP/ggQF", - "hex": "83019f0203ff820405", - "roundtrip": false, - "decoded": [ - 1, - [ - 2, - 3 - ], - [ - 4, - 5 - ] - ] - }, - { - "cbor": "nwECAwQFBgcICQoLDA0ODxAREhMUFRYXGBgYGf8=", - "hex": "9f0102030405060708090a0b0c0d0e0f101112131415161718181819ff", - "roundtrip": false, - "decoded": [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25 - ] - }, - { - "cbor": "v2FhAWFinwID//8=", - "hex": "bf61610161629f0203ffff", - "roundtrip": false, - "decoded": { - "a": 1, - "b": [ - 2, - 3 - ] - } - }, - { - "cbor": "gmFhv2FiYWP/", - "hex": "826161bf61626163ff", - "roundtrip": false, - "decoded": [ - "a", - { - "b": "c" - } - ] - }, - { - "cbor": "v2NGdW71Y0FtdCH/", - "hex": "bf6346756ef563416d7421ff", - "roundtrip": false, - "decoded": { - "Fun": true, - "Amt": -2 - } - } -] diff --git a/codec/test.py b/codec/test.py index 800376f6..8418fee4 100755 --- a/codec/test.py +++ b/codec/test.py @@ -4,14 +4,14 @@ # A Test calls this internally to create the golden files # So it can process them (so we don't have to checkin the files). -# Ensure msgpack-python and cbor are installed first, using: +# Ensure msgpack-python is installed first, using: # sudo apt-get install python-dev # sudo apt-get install python-pip -# pip install --user msgpack-python msgpack-rpc-python cbor +# pip install --user msgpack-python msgpack-rpc-python # Ensure all "string" keys are utf strings (else encoded as bytes) -import cbor, msgpack, msgpackrpc, sys, os, threading +import msgpack, msgpackrpc, sys, os, threading def get_test_data_list(): # get list with all primitive types, and a combo type @@ -72,15 +72,11 @@ def build_test_data(destdir): f = open(os.path.join(destdir, str(i) + '.msgpack.golden'), 'wb') f.write(serialized) f.close() - serialized = cbor.dumps(l[i]) - f = open(os.path.join(destdir, str(i) + '.cbor.golden'), 'wb') - f.write(serialized) - f.close() def doRpcServer(port, stopTimeSec): class EchoHandler(object): def Echo123(self, msg1, msg2, msg3): - return ("1:%s 2:%s 3:%s" % (msg1, msg2, msg3)) + return ("1:%s 2:%s 3:%s" % (msg1.decode('utf8'), msg2.decode('utf8'), msg3.decode('utf8'))) def EchoStruct(self, msg): return ("%s" % msg) @@ -98,15 +94,15 @@ def myStopRpcServer(): def doRpcClientToPythonSvc(port): address = msgpackrpc.Address('127.0.0.1', port) client = msgpackrpc.Client(address, unpack_encoding='utf-8') - print client.call("Echo123", "A1", "B2", "C3") - print client.call("EchoStruct", {"A" :"Aa", "B":"Bb", "C":"Cc"}) + print(client.call("Echo123", "A1", "B2", "C3")) + print(client.call("EchoStruct", {"A" :"Aa", "B":"Bb", "C":"Cc"})) def doRpcClientToGoSvc(port): # print ">>>> port: ", port, " <<<<<" address = msgpackrpc.Address('127.0.0.1', port) client = msgpackrpc.Client(address, unpack_encoding='utf-8') - print client.call("TestRpcInt.Echo123", ["A1", "B2", "C3"]) - print client.call("TestRpcInt.EchoStruct", {"A" :"Aa", "B":"Bb", "C":"Cc"}) + print(client.call("TestRpcInt.Echo123", ["A1", "B2", "C3"])) + print(client.call("TestRpcInt.EchoStruct", {"A" :"Aa", "B":"Bb", "C":"Cc"})) def doMain(args): if len(args) == 2 and args[0] == "testdata": diff --git a/codec/xml.go b/codec/xml.go deleted file mode 100644 index fe4b80c8..00000000 --- a/codec/xml.go +++ /dev/null @@ -1,509 +0,0 @@ -// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved. -// Use of this source code is governed by a MIT license found in the LICENSE file. - -//go:build ignore -// +build ignore - -package codec - -import "reflect" - -/* - -A strict Non-validating namespace-aware XML 1.0 parser and (en|de)coder. - -We are attempting this due to perceived issues with encoding/xml: - - Complicated. It tried to do too much, and is not as simple to use as json. - - Due to over-engineering, reflection is over-used AND performance suffers: - java is 6X faster:http://fabsk.eu/blog/category/informatique/dev/golang/ - even PYTHON performs better: http://outgoing.typepad.com/outgoing/2014/07/exploring-golang.html - -codec framework will offer the following benefits - - VASTLY improved performance (when using reflection-mode or codecgen) - - simplicity and consistency: with the rest of the supported formats - - all other benefits of codec framework (streaming, codegeneration, etc) - -codec is not a drop-in replacement for encoding/xml. -It is a replacement, based on the simplicity and performance of codec. -Look at it like JAXB for Go. - -Challenges: - - Need to output XML preamble, with all namespaces at the right location in the output. - - Each "end" block is dynamic, so we need to maintain a context-aware stack - - How to decide when to use an attribute VS an element - - How to handle chardata, attr, comment EXPLICITLY. - - Should it output fragments? - e.g. encoding a bool should just output true OR false, which is not well-formed XML. - -Extend the struct tag. See representative example: - type X struct { - ID uint8 `codec:"http://ugorji.net/x-namespace xid id,omitempty,toarray,attr,cdata"` - // format: [namespace-uri ][namespace-prefix ]local-name, ... - } - -Based on this, we encode - - fields as elements, BUT - encode as attributes if struct tag contains ",attr" and is a scalar (bool, number or string) - - text as entity-escaped text, BUT encode as CDATA if struct tag contains ",cdata". - -To handle namespaces: - - XMLHandle is denoted as being namespace-aware. - Consequently, we WILL use the ns:name pair to encode and decode if defined, else use the plain name. - - *Encoder and *Decoder know whether the Handle "prefers" namespaces. - - add *Encoder.getEncName(*structFieldInfo). - No one calls *structFieldInfo.indexForEncName directly anymore - - OR better yet: indexForEncName is namespace-aware, and helper.go is all namespace-aware - indexForEncName takes a parameter of the form namespace:local-name OR local-name - - add *Decoder.getStructFieldInfo(encName string) // encName here is either like abc, or h1:nsabc - by being a method on *Decoder, or maybe a method on the Handle itself. - No one accesses .encName anymore - - let encode.go and decode.go use these (for consistency) - - only problem exists for gen.go, where we create a big switch on encName. - Now, we also have to add a switch on strings.endsWith(kName, encNsName) - - gen.go will need to have many more methods, and then double-on the 2 switch loops like: - switch k { - case "abc" : x.abc() - case "def" : x.def() - default { - switch { - case !nsAware: panic(...) - case strings.endsWith(":abc"): x.abc() - case strings.endsWith(":def"): x.def() - default: panic(...) - } - } - } - -The structure below accommodates this: - - type typeInfo struct { - sfi []*structFieldInfo // sorted by encName - sfins // sorted by namespace - sfia // sorted, to have those with attributes at the top. Needed to write XML appropriately. - sfip // unsorted - } - type structFieldInfo struct { - encName - nsEncName - ns string - attr bool - cdata bool - } - -indexForEncName is now an internal helper function that takes a sorted array -(one of ti.sfins or ti.sfi). It is only used by *Encoder.getStructFieldInfo(...) - -There will be a separate parser from the builder. -The parser will have a method: next() xmlToken method. It has lookahead support, -so you can pop multiple tokens, make a determination, and push them back in the order popped. -This will be needed to determine whether we are "nakedly" decoding a container or not. -The stack will be implemented using a slice and push/pop happens at the [0] element. - -xmlToken has fields: - - type uint8: 0 | ElementStart | ElementEnd | AttrKey | AttrVal | Text - - value string - - ns string - -SEE: http://www.xml.com/pub/a/98/10/guide0.html?page=3#ENTDECL - -The following are skipped when parsing: - - External Entities (from external file) - - Notation Declaration e.g. - - Entity Declarations & References - - XML Declaration (assume UTF-8) - - XML Directive i.e. - - Other Declarations: Notation, etc. - - Comment - - Processing Instruction - - schema / DTD for validation: - We are not a VALIDATING parser. Validation is done elsewhere. - However, some parts of the DTD internal subset are used (SEE BELOW). - For Attribute List Declarations e.g. - - We considered using the ATTLIST to get "default" value, but not to validate the contents. (VETOED) - -The following XML features are supported - - Namespace - - Element - - Attribute - - cdata - - Unicode escape - -The following DTD (when as an internal sub-set) features are supported: - - Internal Entities e.g. - AND entities for the set: [<>&"'] - - Parameter entities e.g. - - -At decode time, a structure containing the following is kept - - namespace mapping - - default attribute values - - all internal entities (<>&"' and others written in the document) - -When decode starts, it parses XML namespace declarations and creates a map in the -xmlDecDriver. While parsing, that map continuously gets updated. -The only problem happens when a namespace declaration happens on the node that it defines. -e.g. -To handle this, each Element must be fully parsed at a time, -even if it amounts to multiple tokens which are returned one at a time on request. - -xmlns is a special attribute name. - - It is used to define namespaces, including the default - - It is never returned as an AttrKey or AttrVal. - *We may decide later to allow user to use it e.g. you want to parse the xmlns mappings into a field.* - -Number, bool, null, mapKey, etc can all be decoded from any xmlToken. -This accommodates map[int]string for example. - -It should be possible to create a schema from the types, -or vice versa (generate types from schema with appropriate tags). -This is however out-of-scope from this parsing project. - -We should write all namespace information at the first point that it is referenced in the tree, -and use the mapping for all child nodes and attributes. This means that state is maintained -at a point in the tree. This also means that calls to Decode or MustDecode will reset some state. - -When decoding, it is important to keep track of entity references and default attribute values. -It seems these can only be stored in the DTD components. We should honor them when decoding. - -Configuration for XMLHandle will look like this: - - XMLHandle - DefaultNS string - // Encoding: - NS map[string]string // ns URI to key, used for encoding - // Decoding: in case ENTITY declared in external schema or dtd, store info needed here - Entities map[string]string // map of entity rep to character - - -During encode, if a namespace mapping is not defined for a namespace found on a struct, -then we create a mapping for it using nsN (where N is 1..1000000, and doesn't conflict -with any other namespace mapping). - -Note that different fields in a struct can have different namespaces. -However, all fields will default to the namespace on the _struct field (if defined). - -An XML document is a name, a map of attributes and a list of children. -Consequently, we cannot "DecodeNaked" into a map[string]interface{} (for example). -We have to "DecodeNaked" into something that resembles XML data. - -To support DecodeNaked (decode into nil interface{}), we have to define some "supporting" types: - type Name struct { // Preferred. Less allocations due to conversions. - Local string - Space string - } - type Element struct { - Name Name - Attrs map[Name]string - Children []interface{} // each child is either *Element or string - } -Only two "supporting" types are exposed for XML: Name and Element. - -// ------------------ - -We considered 'type Name string' where Name is like "Space Local" (space-separated). -We decided against it, because each creation of a name would lead to -double allocation (first convert []byte to string, then concatenate them into a string). -The benefit is that it is faster to read Attrs from a map. But given that Element is a value -object, we want to eschew methods and have public exposed variables. - -We also considered the following, where xml types were not value objects, and we used -intelligent accessor methods to extract information and for performance. -*** WE DECIDED AGAINST THIS. *** - type Attr struct { - Name Name - Value string - } - // Element is a ValueObject: There are no accessor methods. - // Make element self-contained. - type Element struct { - Name Name - attrsMap map[string]string // where key is "Space Local" - attrs []Attr - childrenT []string - childrenE []Element - childrenI []int // each child is a index into T or E. - } - func (x *Element) child(i) interface{} // returns string or *Element - -// ------------------ - -Per XML spec and our default handling, white space is always treated as -insignificant between elements, except in a text node. The xml:space='preserve' -attribute is ignored. - -**Note: there is no xml: namespace. The xml: attributes were defined before namespaces.** -**So treat them as just "directives" that should be interpreted to mean something**. - -On encoding, we support indenting aka prettifying markup in the same way we support it for json. - -A document or element can only be encoded/decoded from/to a struct. In this mode: - - struct name maps to element name (or tag-info from _struct field) - - fields are mapped to child elements or attributes - -A map is either encoded as attributes on current element, or as a set of child elements. -Maps are encoded as attributes iff their keys and values are primitives (number, bool, string). - -A list is encoded as a set of child elements. - -Primitives (number, bool, string) are encoded as an element, attribute or text -depending on the context. - -Extensions must encode themselves as a text string. - -Encoding is tough, specifically when encoding mappings, because we need to encode -as either attribute or element. To do this, we need to default to encoding as attributes, -and then let Encoder inform the Handle when to start encoding as nodes. -i.e. Encoder does something like: - - h.EncodeMapStart() - h.Encode(), h.Encode(), ... - h.EncodeMapNotAttrSignal() // this is not a bool, because it's a signal - h.Encode(), h.Encode(), ... - h.EncodeEnd() - -Only XMLHandle understands this, and will set itself to start encoding as elements. - -This support extends to maps. For example, if a struct field is a map, and it has -the struct tag signifying it should be attr, then all its fields are encoded as attributes. -e.g. - - type X struct { - M map[string]int `codec:"m,attr"` // encode keys as attributes named - } - -Question: - - if encoding a map, what if map keys have spaces in them??? - Then they cannot be attributes or child elements. Error. - -Options to consider adding later: - - For attribute values, normalize by trimming beginning and ending white space, - and converting every white space sequence to a single space. - - ATTLIST restrictions are enforced. - e.g. default value of xml:space, skipping xml:XYZ style attributes, etc. - - Consider supporting NON-STRICT mode (e.g. to handle HTML parsing). - Some elements e.g. br, hr, etc need not close and should be auto-closed - ... (see http://www.w3.org/TR/html4/loose.dtd) - An expansive set of entities are pre-defined. - - Have easy way to create a HTML parser: - add a HTML() method to XMLHandle, that will set Strict=false, specify AutoClose, - and add HTML Entities to the list. - - Support validating element/attribute XMLName before writing it. - Keep this behind a flag, which is set to false by default (for performance). - type XMLHandle struct { - CheckName bool - } - -Misc: - -ROADMAP (1 weeks): - - build encoder (1 day) - - build decoder (based off xmlParser) (1 day) - - implement xmlParser (2 days). - Look at encoding/xml for inspiration. - - integrate and TEST (1 days) - - write article and post it (1 day) - -// ---------- MORE NOTES FROM 2017-11-30 ------------ - -when parsing -- parse the attributes first -- then parse the nodes - -basically: -- if encoding a field: we use the field name for the wrapper -- if encoding a non-field, then just use the element type name - - map[string]string ==> abcval... or - val... OR - val1val2... <- PREFERED - []string ==> v1v2... - string v1 ==> v1 - bool true ==> true - float 1.0 ==> 1.0 - ... - - F1 map[string]string ==> abcval... OR - val... OR - val... <- PREFERED - F2 []string ==> v1v2... - F3 bool ==> true - ... - -- a scalar is encoded as: - (value) of type T ==> - (value) of field F ==> -- A kv-pair is encoded as: - (key,value) ==> OR - (key,value) of field F ==> OR -- A map or struct is just a list of kv-pairs -- A list is encoded as sequences of same node e.g. - - - value21 - value22 -- we may have to singularize the field name, when entering into xml, - and pluralize them when encoding. -- bi-directional encode->decode->encode is not a MUST. - even encoding/xml cannot decode correctly what was encoded: - - see https://play.golang.org/p/224V_nyhMS - func main() { - fmt.Println("Hello, playground") - v := []interface{}{"hello", 1, true, nil, time.Now()} - s, err := xml.Marshal(v) - fmt.Printf("err: %v, \ns: %s\n", err, s) - var v2 []interface{} - err = xml.Unmarshal(s, &v2) - fmt.Printf("err: %v, \nv2: %v\n", err, v2) - type T struct { - V []interface{} - } - v3 := T{V: v} - s, err = xml.Marshal(v3) - fmt.Printf("err: %v, \ns: %s\n", err, s) - var v4 T - err = xml.Unmarshal(s, &v4) - fmt.Printf("err: %v, \nv4: %v\n", err, v4) - } - Output: - err: , - s: hello1true - err: , - v2: [] - err: , - s: hello1true2009-11-10T23:00:00Z - err: , - v4: {[ ]} -- -*/ - -// ----------- PARSER ------------------- - -type xmlTokenType uint8 - -const ( - _ xmlTokenType = iota << 1 - xmlTokenElemStart - xmlTokenElemEnd - xmlTokenAttrKey - xmlTokenAttrVal - xmlTokenText -) - -type xmlToken struct { - Type xmlTokenType - Value string - Namespace string // blank for AttrVal and Text -} - -type xmlParser struct { - r decReader - toks []xmlToken // list of tokens. - ptr int // ptr into the toks slice - done bool // nothing else to parse. r now returns EOF. -} - -func (x *xmlParser) next() (t *xmlToken) { - // once x.done, or x.ptr == len(x.toks) == 0, then return nil (to signify finish) - if !x.done && len(x.toks) == 0 { - x.nextTag() - } - // parses one element at a time (into possible many tokens) - if x.ptr < len(x.toks) { - t = &(x.toks[x.ptr]) - x.ptr++ - if x.ptr == len(x.toks) { - x.ptr = 0 - x.toks = x.toks[:0] - } - } - return -} - -// nextTag will parses the next element and fill up toks. -// It set done flag if/once EOF is reached. -func (x *xmlParser) nextTag() { - // TODO: implement. -} - -// ----------- ENCODER ------------------- - -type xmlEncDriver struct { - e *Encoder - w encWriter - h *XMLHandle - b [64]byte // scratch - bs []byte // scratch - // s jsonStack - noBuiltInTypes -} - -// ----------- DECODER ------------------- - -type xmlDecDriver struct { - d *Decoder - h *XMLHandle - r decReader // *bytesDecReader decReader - ct valueType // container type. one of unset, array or map. - bstr [8]byte // scratch used for string \UXXX parsing - b [64]byte // scratch - - // wsSkipped bool // whitespace skipped - - // s jsonStack - - noBuiltInTypes -} - -// DecodeNaked will decode into an XMLNode - -// XMLName is a value object representing a namespace-aware NAME -type XMLName struct { - Local string - Space string -} - -// XMLNode represents a "union" of the different types of XML Nodes. -// Only one of fields (Text or *Element) is set. -type XMLNode struct { - Element *Element - Text string -} - -// XMLElement is a value object representing an fully-parsed XML element. -type XMLElement struct { - Name Name - Attrs map[XMLName]string - // Children is a list of child nodes, each being a *XMLElement or string - Children []XMLNode -} - -// ----------- HANDLE ------------------- - -type XMLHandle struct { - BasicHandle - textEncodingType - - DefaultNS string - NS map[string]string // ns URI to key, for encoding - Entities map[string]string // entity representation to string, for encoding. -} - -func (h *XMLHandle) newEncDriver(e *Encoder) encDriver { - return &xmlEncDriver{e: e, w: e.w, h: h} -} - -func (h *XMLHandle) newDecDriver(d *Decoder) decDriver { - // d := xmlDecDriver{r: r.(*bytesDecReader), h: h} - hd := xmlDecDriver{d: d, r: d.r, h: h} - hd.n.bytes = d.b[:] - return &hd -} - -func (h *XMLHandle) SetInterfaceExt(rt reflect.Type, tag uint64, ext InterfaceExt) (err error) { - return h.SetExt(rt, tag, &extWrapper{bytesExtFailer{}, ext}) -} - -var _ decDriver = (*xmlDecDriver)(nil) -var _ encDriver = (*xmlEncDriver)(nil) diff --git a/go.mod b/go.mod index 41cce53b..f7578b4b 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,12 @@ module github.com/hashicorp/go-msgpack/v2 go 1.19 require ( - bitbucket.org/bodhisnarkva/cbor v0.0.0-20170201010848-113f42203c94 github.com/Sereal/Sereal v0.0.0-20221130110801-16a4f76670cd github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892 github.com/json-iterator/go v1.1.12 github.com/mailru/easyjson v0.7.7 github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 github.com/tinylib/msgp v1.1.6 - github.com/ugorji/go/codec v1.2.7 golang.org/x/tools v0.3.0 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 gopkg.in/vmihailenco/msgpack.v2 v2.9.2 @@ -18,17 +16,18 @@ require ( require ( github.com/DataDog/zstd v1.5.2 // indirect - github.com/golang/protobuf v1.3.1 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/philhofer/fwd v1.1.1 // indirect golang.org/x/mod v0.7.0 // indirect golang.org/x/net v0.2.0 // indirect - golang.org/x/sys v0.2.0 // indirect + golang.org/x/sys v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index de8b8c21..9be0b672 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -bitbucket.org/bodhisnarkva/cbor v0.0.0-20170201010848-113f42203c94 h1:mIPAugcfnbnaazg3ttUVbHbJaY+e3aUf4Wr7lONml5I= -bitbucket.org/bodhisnarkva/cbor v0.0.0-20170201010848-113f42203c94/go.mod h1:pl6BcNYTFXkbj1OCW5EIJkgrzmmEFwU6oaelsR/9pMQ= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Sereal/Sereal v0.0.0-20221130110801-16a4f76670cd h1:rP6LH3aVJTIxgTA3q79sSfnt8DvOlt17IRAklRBN+xo= @@ -9,10 +7,13 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892 h1:qg9VbHo1TlL0KDM0vYvBG9EY0X0Yku5WYIPoFWt8f6o= github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -27,8 +28,9 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= @@ -42,9 +44,6 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw= github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= -github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= -github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= -github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -64,8 +63,8 @@ golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -76,9 +75,14 @@ golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/msgpack.org.md b/msgpack.org.md deleted file mode 100644 index d5ebe71d..00000000 --- a/msgpack.org.md +++ /dev/null @@ -1,47 +0,0 @@ -**MessagePack and [Binc](http://github.com/ugorji/binc) Codec for [Go](http://golang.org) Language.** - -*A High Performance, Feature-Rich, Idiomatic encode/decode and rpc library*. - -To install: - - go get github.com/ugorji/go/codec - -Source: [http://github.com/ugorji/go] -Online documentation: [http://godoc.org/github.com/ugorji/go/codec] - -Typical usage: - -```go - // create and use decoder/encoder - var ( - v interface{} // value to decode/encode into - r io.Reader - w io.Writer - b []byte - mh codec.MsgpackHandle - ) - - dec = codec.NewDecoder(r, &mh) - dec = codec.NewDecoderBytes(b, &mh) - err = dec.Decode(&v) - - enc = codec.NewEncoder(w, &mh) - enc = codec.NewEncoderBytes(&b, &mh) - err = enc.Encode(v) - - //RPC Server - go func() { - for { - conn, err := listener.Accept() - rpcCodec := codec.GoRpc.ServerCodec(conn, h) - //OR rpcCodec := codec.MsgpackSpecRpc.ServerCodec(conn, h) - rpc.ServeCodec(rpcCodec) - } - }() - - //RPC Communication (client side) - conn, err = net.Dial("tcp", "localhost:5555") - rpcCodec := codec.GoRpc.ClientCodec(conn, h) - //OR rpcCodec := codec.MsgpackSpecRpc.ClientCodec(conn, h) - client := rpc.NewClientWithCodec(rpcCodec) -```