codec: Add canonical encoding support, handle EOF (as non-error) if R…

…ead successful, and export CborStreamBreak.

Canonical representation means that encoding a value will always result in the same sequence of bytes.
This mostly will apply to maps. In this case, codec will do more work to encode the
map keys out of band, and then sort them, before writing out the map to the stream.

Canonical flag is only honored within the standard runtime-introspection (reflection-based)
mode of encoding. It is ignored by codecgen (code generation). Also, if canonical flag is true,
then fast-path encoding of maps is skipped.

There is a slight performance hit if Canonical flag is on, as we have to encode the keys
out-of-band, and then sort them, before encoding the whole map.

Canonical mode is turned on via a flag in EncodeOptions (handler).

Also, export CborStreamBreak as a convenience and for symmetry (as other stream constants were exported).

Also, handle EOF if Read was successful.
If a successful Read happened, but an EOF was returned at same time,
we do not treat that as an error.

Fixes #56
Fixes #58
Fixes #59

codec/0doc.go

@@ -58,6 +58,8 @@ Rich Feature Set includes:
   - Support binary (e.g. messagepack, cbor) and text (e.g. json) formats
   - Support indefinite-length formats to enable true streaming 
     (for formats which support it e.g. json, cbor)
+  - Support canonical encoding, where a value is ALWAYS encoded as same sequence of bytes.
+    This mostly applies to maps, where iteration order is non-deterministic.
   - NIL in data stream decoded as zero value
   - Never silently skip data when decoding.
     User decides whether to return an error or silently skip data when keys or indexes

codec/README.md

@@ -59,6 +59,8 @@ Rich Feature Set includes:
   - Support binary (e.g. messagepack, cbor) and text (e.g. json) formats
   - Support indefinite-length formats to enable true streaming 
     (for formats which support it e.g. json, cbor)
+  - Support canonical encoding, where a value is ALWAYS encoded as same sequence of bytes.
+    This mostly applies to maps, where iteration order is non-deterministic.
   - NIL in data stream decoded as zero value
   - Never silently skip data when decoding.
     User decides whether to return an error or silently skip data when keys or indexes

codec/cbor.go

@@ -40,7 +40,7 @@ const (
 	CborStreamString      = 0x7f
 	CborStreamArray       = 0x9f
 	CborStreamMap         = 0xbf
-	cborStreamBreak       = 0xff
+	CborStreamBreak       = 0xff
 )
 
 const (

codec/codec_test.go

@@ -63,6 +63,7 @@ var (
 	testInitDebug      bool
 	testUseIoEncDec    bool
 	testStructToArray  bool
+	testCanonical      bool
 	testWriteNoSymbols bool
 	testSkipIntf       bool
 
@@ -95,6 +96,7 @@ func testInitFlags() {
 	flag.BoolVar(&testUseIoEncDec, "ti", false, "Use IO Reader/Writer for Marshal/Unmarshal")
 	flag.BoolVar(&testStructToArray, "ts", false, "Set StructToArray option")
 	flag.BoolVar(&testWriteNoSymbols, "tn", false, "Set NoSymbols option")
+	flag.BoolVar(&testCanonical, "tc", false, "Set Canonical option")
 	flag.BoolVar(&testSkipIntf, "tf", false, "Skip Interfaces")
 }
 
@@ -267,17 +269,26 @@ func testInit() {
 		fmt.Printf("====> depth: %v, ts: %#v\n", 2, ts0)
 	}
 
+	testJsonH.Canonical = testCanonical
+	testCborH.Canonical = testCanonical
+	testSimpleH.Canonical = testCanonical
+	testBincH.Canonical = testCanonical
+	testMsgpackH.Canonical = testCanonical
+
 	testJsonH.StructToArray = testStructToArray
 	testCborH.StructToArray = testStructToArray
 	testSimpleH.StructToArray = testStructToArray
 	testBincH.StructToArray = testStructToArray
+	testMsgpackH.StructToArray = testStructToArray
+
+	testMsgpackH.RawToString = true
+
 	if testWriteNoSymbols {
 		testBincH.AsSymbols = AsSymbolNone
 	} else {
 		testBincH.AsSymbols = AsSymbolAll
 	}
-	testMsgpackH.StructToArray = testStructToArray
-	testMsgpackH.RawToString = true
+
 	// testMsgpackH.AddExt(byteSliceTyp, 0, testMsgpackH.BinaryEncodeExt, testMsgpackH.BinaryDecodeExt)
 	// testMsgpackH.AddExt(timeTyp, 1, testMsgpackH.TimeEncodeExt, testMsgpackH.TimeDecodeExt)
 	timeEncExt := func(rv reflect.Value) (bs []byte, err error) {
@@ -1100,3 +1111,7 @@ func TestBincUnderlyingType(t *testing.T) {
 //   - interfaces: textMarshaler, binaryMarshaler, codecSelfer
 //   - struct tags:
 //     on anonymous fields, _struct (all fields), etc
+//   - codecgen of struct containing channels.
+//
+//   Cleanup tests:
+//   - The are brittle in their handling of validation and skipping

codec/decode.go

@@ -141,6 +141,9 @@ func (z *ioDecByteScanner) Read(p []byte) (n int, err error) {
 	}
 	n, err = z.r.Read(p)
 	if n > 0 {
+		if err == io.EOF && n == len(p) {
+			err = nil // read was successful, so postpone EOF (till next time)
+		}
 		z.l = p[n-1]
 		z.ls = 2
 	}
@@ -154,6 +157,9 @@ func (z *ioDecByteScanner) ReadByte() (c byte, err error) {
 	n, err := z.Read(z.b[:])
 	if n == 1 {
 		c = z.b[0]
+		if err == io.EOF {
+			err = nil // read was successful, so postpone EOF (till next time)
+		}
 	}
 	return
 }
@@ -707,7 +713,7 @@ func (f decFnInfo) kSlice(rv reflect.Value) {
 	for rtelem.Kind() == reflect.Ptr {
 		rtelem = rtelem.Elem()
 	}
-	fn := d.getDecFn(rtelem, true)
+	fn := d.getDecFn(rtelem, true, true)
 
 	rv0 := rv
 	rvChanged := false
@@ -816,10 +822,10 @@ func (f decFnInfo) kMap(rv reflect.Value) {
 	var xtyp reflect.Type
 	for xtyp = ktype; xtyp.Kind() == reflect.Ptr; xtyp = xtyp.Elem() {
 	}
-	keyFn = d.getDecFn(xtyp, true)
+	keyFn = d.getDecFn(xtyp, true, true)
 	for xtyp = vtype; xtyp.Kind() == reflect.Ptr; xtyp = xtyp.Elem() {
 	}
-	valFn = d.getDecFn(xtyp, true)
+	valFn = d.getDecFn(xtyp, true, true)
 	// for j := 0; j < containerLen; j++ {
 	if containerLen > 0 {
 		for j := 0; j < containerLen; j++ {
@@ -1153,7 +1159,7 @@ func (d *Decoder) decode(iv interface{}) {
 
 	default:
 		if !fastpathDecodeTypeSwitch(iv, d) {
-			d.decodeI(iv, true, false, false)
+			d.decodeI(iv, true, false, false, false)
 		}
 	}
 }
@@ -1180,22 +1186,22 @@ func (d *Decoder) preDecodeValue(rv reflect.Value, tryNil bool) (rv2 reflect.Val
 	return rv, true
 }
 
-func (d *Decoder) decodeI(iv interface{}, checkPtr, tryNil, decFnCheckAll bool) {
+func (d *Decoder) decodeI(iv interface{}, checkPtr, tryNil, checkFastpath, checkCodecSelfer bool) {
 	rv := reflect.ValueOf(iv)
 	if checkPtr {
 		d.chkPtrValue(rv)
 	}
 	rv, proceed := d.preDecodeValue(rv, tryNil)
 	if proceed {
-		fn := d.getDecFn(rv.Type(), decFnCheckAll)
+		fn := d.getDecFn(rv.Type(), checkFastpath, checkCodecSelfer)
 		fn.f(fn.i, rv)
 	}
 }
 
 func (d *Decoder) decodeValue(rv reflect.Value, fn decFn) {
 	if rv, proceed := d.preDecodeValue(rv, true); proceed {
 		if fn.f == nil {
-			fn = d.getDecFn(rv.Type(), true)
+			fn = d.getDecFn(rv.Type(), true, true)
 		}
 		fn.f(fn.i, rv)
 	}
@@ -1204,13 +1210,13 @@ func (d *Decoder) decodeValue(rv reflect.Value, fn decFn) {
 func (d *Decoder) decodeValueNotNil(rv reflect.Value, fn decFn) {
 	if rv, proceed := d.preDecodeValue(rv, false); proceed {
 		if fn.f == nil {
-			fn = d.getDecFn(rv.Type(), true)
+			fn = d.getDecFn(rv.Type(), true, true)
 		}
 		fn.f(fn.i, rv)
 	}
 }
 
-func (d *Decoder) getDecFn(rt reflect.Type, checkAll bool) (fn decFn) {
+func (d *Decoder) getDecFn(rt reflect.Type, checkFastpath, checkCodecSelfer bool) (fn decFn) {
 	rtid := reflect.ValueOf(rt).Pointer()
 
 	// retrieve or register a focus'ed function for this type
@@ -1247,7 +1253,7 @@ func (d *Decoder) getDecFn(rt reflect.Type, checkAll bool) (fn decFn) {
 	//
 	// NOTE: if decoding into a nil interface{}, we return a non-nil
 	// value except even if the container registers a length of 0.
-	if checkAll && ti.cs {
+	if checkCodecSelfer && ti.cs {
 		fi.decFnInfoX = &decFnInfoX{d: d, ti: ti}
 		fn.f = (decFnInfo).selferUnmarshal
 	} else if rtid == rawExtTypId {
@@ -1269,7 +1275,7 @@ func (d *Decoder) getDecFn(rt reflect.Type, checkAll bool) (fn decFn) {
 		fn.f = (decFnInfo).textUnmarshal
 	} else {
 		rk := rt.Kind()
-		if fastpathEnabled && checkAll && (rk == reflect.Map || rk == reflect.Slice) {
+		if fastpathEnabled && checkFastpath && (rk == reflect.Map || rk == reflect.Slice) {
 			if rt.PkgPath() == "" {
 				if idx := fastpathAV.index(rtid); idx != -1 {
 					fi.decFnInfoX = &decFnInfoX{d: d, ti: ti}