diff --git a/README.md b/README.md index 0c7aaf0..23e6880 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ The Rarefied Encoding (Compressible) for Interchange (REZI) library performs binary marshaling of data to REZI-format bytes. It can encode and decode most -simple (non-struct) built-in Go types to bytes, and handles customization of -decoding and encoding of user-defined types that implement +simple built-in Go types and structs that contain those types to bytes. It +further allows customization of decoding and encoding of user-defined types via `encoding.BinaryMarshaler` or `encoding.TextMarshaler`. All data is encoded in a deterministic fashion, or as deterministically as @@ -204,29 +204,137 @@ integer types, or one of the built-in float types. REZI can also handle encoding and decoding pointers to any supported type, with any level of indirection. -On top of all of the above, REZI supports any type whose underlying type is -supported. +On top of all of the above, REZI automatically supports any type whose +underlying type is supported, as well as any struct whose exported fields are +all of supported types. + +#### Struct Support + +Much like the `json` package, REZI can encode and decode most simple structs +out of the box without requiring any further customization. Simple in this case +means that all of its exported fields are a supported type. As only the exported +fields are encoded and decoded to bytes, it's okay if an unexported field is of +an unsupported type. + +```golang + +// AnimalInfo is fully supported; all fields will be encoded and decoded. +type AnimalInfo struct { + Name string + Taxonomy []string + AverageAge int +} + +// Animal is *not* supported; despite field Info being of a supported type, +// Counter is a channel, which is unsupported. +type Animal struct { + Info AnimalInfo + Counter chan int +} + +// HiddenCounterAnimal is supported. Even though field counter is of an +// unsupported type, it is unexported and so it will be ignored. +type HiddenCounterAnimal struct { + Info AnimalInfo + counter chan int +} +``` + +Unexported fields of a struct are ignored when encoding and decoding. Any struct +that has unexported fields will keep their original values if a pointer to that +struct is passed in to be decoded. + +```golang +type Player struct { + Name string + Classpect string + + echeladder string +} + +john := Player{Name: "John Egbert", Classpect: "Heir of Breath", echeladder: "Plucky Tot"} + +// the encoded bytes will only contain Name and Classpect; echeladder is not +// exported so it is ignored +data, err := rezi.Enc(john) +if err != nil { + panic(err) +} + +// now we will decode the bytes to two structs, one with the unexported member +// pre-set +var playerWithoutEcheladder Player +var playerWithEcheladder Player = Player{echeladder: "Plucky Tot"} + +_, err = rezi.Dec(data, &playerWithoutEcheladder) +if err != nil { + panic(err) +} +_, err = rezi.Dec(data, &playerWithEcheladder) +if err != nil { + panic(err) +} + +fmt.Println(playerWithoutEcheladder.echeladder) // "" +fmt.Println(playerWithEcheladder.echeladder) // "Plucky Tot" +``` + +Embedded structs within structs are supported if the embedded struct type is +exported; this is because it will be turned into a field with the same name as +the embedded type, and if it is exported, the field name will correspondingly +be exported. Likewise, embedded structs whose type is unexported will be ignored +during encoding and will not be encoded to. + +```golang +type InternalRecord struct { + ID int + Location string +} + +type secret struct { + BigSecret string +} + +// All fields of Employee will be marshaled and unmarshaled to; InternalRecord +// is exported +type Employee struct { + InternalRecord + Name string +} + +// Only Name will be encoded and decoded to; secret is an unexported type. +type KeyData struct { + secret + Name string +} +``` + +If any of the above limitations are a concern, you can customize the encoding of +user-defined types by implementing one of the marshaler types +`encoding.BinaryMarshaler` or `encoding.TextMarshaler` (and their corresponding +unmarshler interfaces for decoding) as described in the next section. + +### Customizing Encoding And Decoding -#### User-Defined Types REZI supports encoding any custom type that implements `encoding.BinaryMarshaler`, and it supports decoding any custom type that implements `encoding.BinaryUnmarshaler` with a pointer receiver. In fact, the lack of built-in facilities in Go for binary encoding of user-defined types is partially why REZI exists. -REZI does not perform any automatic inference of a user-defined struct type's -encoding such as what the `json` library is capable of. User-defined types that -do not implement BinaryMarshaler or TextMarshaler are only supported for -encoding if their underlying type is one supported by REZI, and vice-versa for -the corresponding unmarshal methods when decoding. - -Within the `MarshalBinary` method, you can encode the data in whichever format -you wish, though these examples will have that function use REZI to encode the -members of the types. The contents of the slice that MarshalBinary returns are -completely opaque to REZI, which will consider only the slice's length. Do note -that this means that returning a nil slice or an empty but initialized slice -will both be interpreted the same by REZI and will not result in different -encodings. +REZI can perform automatic inference of a user-defined struct type's encoding, +similar to what the `json` library is capable of. User-defined types that +do not implement BinaryMarshaler or TextMarshaler are supported for encoding if +their underlying type is one supported by REZI, or if it is a struct type, if +all of its exported fields are supported, and vice-versa for decoding. + +Within the `MarshalBinary` method, you can customize encoding the data to +whichever format you wish, though these examples will have that function use +REZI to encode the members of the types. The contents of the slice that +MarshalBinary returns are completely opaque to REZI, which will consider only +the slice's length. Do note that this means that returning a nil slice or an +empty but initialized slice will both be interpreted the same by REZI and will +not result in different encodings. ```golang // Person is an example of a user-defined type that REZI can encode and decode. diff --git a/arrays_test.go b/arrays_test.go index 7d6b006..386ae24 100644 --- a/arrays_test.go +++ b/arrays_test.go @@ -4,6 +4,7 @@ package rezi import ( "math" + "sync" "testing" "github.com/stretchr/testify/assert" @@ -235,6 +236,62 @@ func Test_Enc_Array_NoIndirection(t *testing.T) { assert.Equal(expect, actual) }) + t.Run("[2]struct", func(t *testing.T) { + // setup + assert := assert.New(t) + var ( + input = [2]testStructManyFields{ + {Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true, hidden: make(chan int, 3), enabled: &sync.Mutex{}, inc: 48}, + {Name: "ROSE", Value: 12, Factor: 0.00390625, Enabled: false, hidden: make(chan int), enabled: nil, inc: 12}, + } + + expect = []byte{ + 0x01, 0x64, // len=100 + + // "KANAYA" struct: + + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + + // "ROSE" struct: + + 0x01, 0x2f, // struct len=47 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x00, // false + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0x70, // 0.00390625 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x04, 0x52, 0x4f, 0x53, 0x45, // "ROSE" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x0c, // 12 + } + ) + + // execute + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expect, actual) + }) + t.Run("[3]text", func(t *testing.T) { // setup assert := assert.New(t) @@ -1026,6 +1083,129 @@ func Test_Enc_Array_ValueIndirection(t *testing.T) { assert.Equal(expect, actual) }) + t.Run("[2]*struct, all non-nil", func(t *testing.T) { + // setup + assert := assert.New(t) + var ( + input = [2]*testStructManyFields{ + {Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true, hidden: make(chan int, 3), enabled: &sync.Mutex{}, inc: 48}, + {Name: "ROSE", Value: 12, Factor: 0.00390625, Enabled: false, hidden: make(chan int), enabled: nil, inc: 12}, + } + + expect = []byte{ + 0x01, 0x64, // len=100 + + // "KANAYA" struct: + + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + + // "ROSE" struct: + + 0x01, 0x2f, // struct len=47 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x00, // false + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0x70, // 0.00390625 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x04, 0x52, 0x4f, 0x53, 0x45, // "ROSE" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x0c, // 12 + } + ) + + // execute + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expect, actual) + }) + + t.Run("[2]*struct, one nil", func(t *testing.T) { + // setup + assert := assert.New(t) + var ( + input = [2]*testStructManyFields{ + {Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true, hidden: make(chan int, 3), enabled: &sync.Mutex{}, inc: 48}, + } + + expect = []byte{ + 0x01, 0x34, // len=52 + + // "KANAYA" struct: + + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + + // "ROSE" struct: + + 0xa0, // nil + } + ) + + // execute + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expect, actual) + }) + + t.Run("[2]*struct, all nil", func(t *testing.T) { + // setup + assert := assert.New(t) + var ( + input = [2]*testStructManyFields{} + + expect = []byte{ + 0x01, 0x02, // len=2 + + 0xa0, + 0xa0, + } + ) + + // execute + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expect, actual) + }) + t.Run("[3]*map[string]bool, all non-nil", func(t *testing.T) { // setup assert := assert.New(t) @@ -1584,6 +1764,65 @@ func Test_Dec_Array_NoIndirection(t *testing.T) { assert.Equal(expect, actual) }) + t.Run("[2]struct", func(t *testing.T) { + assert := assert.New(t) + + var ( + input = []byte{ + 0x01, 0x64, // len=100 + + // "KANAYA" struct: + + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + + // "ROSE" struct: + + 0x01, 0x2f, // struct len=47 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x00, // false + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0x70, // 0.00390625 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x04, 0x52, 0x4f, 0x53, 0x45, // "ROSE" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x0c, // 12 + } + expect = [2]testStructManyFields{ + {Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true, hidden: nil, enabled: nil, inc: 0}, + {Name: "ROSE", Value: 12, Factor: 0.00390625, Enabled: false, hidden: nil, enabled: nil, inc: 0}, + } + expectConsumed = 102 + ) + + // execute + var actual [2]testStructManyFields + consumed, err := Dec(input, &actual) + + // assert + if !assert.NoError(err) { + return + } + + assert.Equal(expectConsumed, consumed) + assert.Equal(expect, actual) + }) + t.Run("[3]map[string]bool", func(t *testing.T) { // setup assert := assert.New(t) @@ -2443,6 +2682,138 @@ func Test_Dec_Array_ValueIndirection(t *testing.T) { assert.Equal(expect, actual) }) + t.Run("[2]*struct, all non-nil", func(t *testing.T) { + // setup + assert := assert.New(t) + + var ( + input = []byte{ + 0x01, 0x64, // len=100 + + // "KANAYA" struct: + + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + + // "ROSE" struct: + + 0x01, 0x2f, // struct len=47 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x00, // false + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0x70, // 0.00390625 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x04, 0x52, 0x4f, 0x53, 0x45, // "ROSE" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x0c, // 12 + } + expect = [2]*testStructManyFields{ + {Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true, hidden: nil, enabled: nil, inc: 0}, + {Name: "ROSE", Value: 12, Factor: 0.00390625, Enabled: false, hidden: nil, enabled: nil, inc: 0}, + } + expectConsumed = 102 + ) + + // execute + var actual [2]*testStructManyFields + consumed, err := Dec(input, &actual) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expectConsumed, consumed) + assert.Equal(expect, actual) + }) + + t.Run("[2]*struct, one nil", func(t *testing.T) { + // setup + assert := assert.New(t) + + var ( + input = []byte{ + 0x01, 0x34, // len=52 + + // "KANAYA" struct: + + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + + // "ROSE" struct: + + 0xa0, // nil + } + expect = [2]*testStructManyFields{ + {Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true, hidden: nil, enabled: nil, inc: 0}, + } + expectConsumed = 54 + ) + + // execute + var actual [2]*testStructManyFields + consumed, err := Dec(input, &actual) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expectConsumed, consumed) + assert.Equal(expect, actual) + }) + + t.Run("[2]*struct, all nil", func(t *testing.T) { + // setup + assert := assert.New(t) + + var ( + input = []byte{ + 0x01, 0x02, // len=2 + + 0xa0, + 0xa0, + } + expect = [2]*testStructManyFields{} + expectConsumed = 4 + ) + + // execute + var actual [2]*testStructManyFields + consumed, err := Dec(input, &actual) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expectConsumed, consumed) + assert.Equal(expect, actual) + }) + t.Run("[3]*map[string]bool, all non-nil", func(t *testing.T) { // setup assert := assert.New(t) diff --git a/errors_test.go b/errors_test.go index 622b1da..4d50d14 100644 --- a/errors_test.go +++ b/errors_test.go @@ -3,6 +3,7 @@ package rezi import ( "errors" "fmt" + "reflect" "strings" "testing" @@ -399,6 +400,96 @@ func Test_reziError_totalOffset_binary(t *testing.T) { } } +func Test_reziError_totalOffset_struct(t *testing.T) { + refName := reflect.ValueOf(testStructMultiMember{}).Type().Name() + testCases := []struct { + name string + input []byte + expectTotalOffset int + expectErrText string + }{ + { + name: "decode struct: bad byte count", + input: []byte{ + /* byte count = 27 */ 0x01, 0x1b, + + /* "Name" */ 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, + /* "KANAYA" */ 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, + + /* "Value" */ 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, + /* 8 */ 0x01, 0x08, + }, + expectTotalOffset: 2, + expectErrText: "decoded " + refName + " byte count", + }, + { + name: "decode struct: string error", + input: []byte{ + /* byte count = 24 */ 0x01, 0x18, + + /* "Name" */ 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, + 0x41, 0x80, 0x04, 0x41, 0x42, 0x43, 0xc3, // "ABC" followed by invalid seq + + /* "Value" */ 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, + /* 8 */ 0x01, 0x08, + }, + expectTotalOffset: 15, + expectErrText: refName + ".Name: invalid UTF-8 encoding", + }, + { + name: "decode struct: number error", + input: []byte{ + /* byte count = 25 */ 0x01, 0x19, + + /* "Name" */ 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, + /* "KANAYA" */ 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, + + /* "Value" */ 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x01, // a length but no actual num + }, + expectTotalOffset: 27, + expectErrText: refName + ".Value: decoded int byte count", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert := assert.New(t) + + var dest testStructMultiMember + _, err := Dec(tc.input, &dest) + if !assert.Error(err) { + return + } + + // convert it to known type reziError + rErr, ok := err.(reziError) + if !assert.Truef(ok, "Dec returned non-reziErr: %v", err) { + return + } + + // check if the offset is valid + actual, ok := rErr.totalOffset() + if !assert.Truef(ok, "Dec returned reziErr with no offset: %v", err) { + return + } + + // assert the offset is the expected + assert.Equal(tc.expectTotalOffset, actual) + + // and finally, check the err output + expectOffsetHex := fmt.Sprintf("%x", tc.expectTotalOffset) + if len(expectOffsetHex)%2 != 0 { + expectOffsetHex = "0" + expectOffsetHex + } + lowerMsgAct := strings.ToUpper(rErr.Error()) + lowerMsgExp := strings.ToUpper(tc.expectErrText) + assert.Contains(rErr.Error(), expectOffsetHex, "message does not contain offset: %q", rErr.Error()) + assert.Contains(lowerMsgAct, lowerMsgExp, "message does not contain %q: %q", tc.expectErrText, rErr.Error()) + }) + } +} + func Test_reziError_totalOffset_text(t *testing.T) { testCases := []struct { name string diff --git a/maps.go b/maps.go index 3ed2ac5..d216817 100644 --- a/maps.go +++ b/maps.go @@ -7,7 +7,7 @@ import ( "sort" ) -// ti must containt a main type of tIntegral, tBool, or tString +// ti must containt a main type of mtIntegral, mtBool, mtFloat, or mtString type sortableMapKeys struct { keys []reflect.Value ti typeInfo @@ -130,7 +130,10 @@ func decCheckedMap(data []byte, v interface{}, ti typeInfo) (int, error) { func(t reflect.Type) bool { return t.Kind() == reflect.Pointer && t.Elem().Kind() == reflect.Map }, - decMap, + func(b []byte, i interface{}) (interface{}, int, error) { + decN, err := decMap(b, i) + return nil, decN, err + }, )) if err != nil { return n, err @@ -145,7 +148,7 @@ func decCheckedMap(data []byte, v interface{}, ti typeInfo) (int, error) { func decMap(data []byte, v interface{}) (int, error) { var totalConsumed int - toConsume, n, err := decInt[tLen](data) + toConsume, _, n, err := decInt[tLen](data) if err != nil { return 0, errorDecf(0, "decode byte count: %s", err) } diff --git a/maps_test.go b/maps_test.go index b2c5e06..76d36c8 100644 --- a/maps_test.go +++ b/maps_test.go @@ -2,6 +2,7 @@ package rezi import ( "net/http" + "sync" "testing" "github.com/stretchr/testify/assert" @@ -208,6 +209,64 @@ func Test_Enc_Map_NoIndirection(t *testing.T) { assert.Equal(expect, actual) }) + t.Run("map[int]struct", func(t *testing.T) { + // setup + assert := assert.New(t) + var ( + input = map[int]testStructManyFields{ + 4: {Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true, hidden: make(chan int, 3), enabled: &sync.Mutex{}, inc: 48}, + 2: {Name: "ROSE", Value: 12, Factor: 0.00390625, Enabled: false, hidden: make(chan int), enabled: nil, inc: 12}, + } + + expect = []byte{ + 0x01, 0x68, // len=104 + + 0x01, 0x02, // 2: + // "ROSE" struct: + + 0x01, 0x2f, // struct len=47 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x00, // false + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0x70, // 0.00390625 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x04, 0x52, 0x4f, 0x53, 0x45, // "ROSE" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x0c, // 12 + + 0x01, 0x04, // 4: + // "KANAYA" struct: + + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + } + ) + + // execute + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expect, actual) + }) + t.Run("map[int]float64", func(t *testing.T) { // setup assert := assert.New(t) @@ -956,6 +1015,140 @@ func Test_Enc_Map_ValueIndirection(t *testing.T) { assert.Equal(expect, actual) }) + t.Run("map[int]*struct, all non-nil", func(t *testing.T) { + // setup + assert := assert.New(t) + var ( + input = map[int]*testStructManyFields{ + 4: {Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true, hidden: make(chan int, 3), enabled: &sync.Mutex{}, inc: 48}, + 2: {Name: "ROSE", Value: 12, Factor: 0.00390625, Enabled: false, hidden: make(chan int), enabled: nil, inc: 12}, + } + + expect = []byte{ + 0x01, 0x68, // len=104 + + 0x01, 0x02, // 2: + // "ROSE" struct: + + 0x01, 0x2f, // struct len=47 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x00, // false + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0x70, // 0.00390625 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x04, 0x52, 0x4f, 0x53, 0x45, // "ROSE" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x0c, // 12 + + 0x01, 0x04, // 4: + // "KANAYA" struct: + + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + } + ) + + // execute + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expect, actual) + }) + + t.Run("map[int]*struct, one nil", func(t *testing.T) { + // setup + assert := assert.New(t) + var ( + input = map[int]*testStructManyFields{ + 4: nil, + 2: {Name: "ROSE", Value: 12, Factor: 0.00390625, Enabled: false, hidden: make(chan int), enabled: nil, inc: 12}, + } + + expect = []byte{ + 0x01, 0x36, // len=54 + + 0x01, 0x02, // 2: + // "ROSE" struct: + + 0x01, 0x2f, // struct len=47 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x00, // false + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0x70, // 0.00390625 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x04, 0x52, 0x4f, 0x53, 0x45, // "ROSE" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x0c, // 12 + + 0x01, 0x04, // 4: + // "KANAYA" struct: + + 0xa0, // nil + } + ) + + // execute + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expect, actual) + }) + + t.Run("map[int]*struct, all nil", func(t *testing.T) { + // setup + assert := assert.New(t) + var ( + input = map[int]*testStructManyFields{ + 4: nil, + 2: nil, + } + + expect = []byte{ + 0x01, 0x06, // len=6 + + 0x01, 0x02, // 2: + 0xa0, + + 0x01, 0x04, // 4: + 0xa0, // nil + } + ) + + // execute + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expect, actual) + }) + t.Run("map[int]*bool, all non-nil", func(t *testing.T) { // setup assert := assert.New(t) @@ -1640,6 +1833,68 @@ func Test_Dec_Map_NoIndirection(t *testing.T) { assert.Equal(expect, actual) }) + t.Run("map[int]struct", func(t *testing.T) { + // setup + assert := assert.New(t) + + var ( + input = []byte{ + 0x01, 0x68, // len=104 + + 0x01, 0x02, // 2: + // "ROSE" struct: + + 0x01, 0x2f, // struct len=47 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x00, // false + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0x70, // 0.00390625 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x04, 0x52, 0x4f, 0x53, 0x45, // "ROSE" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x0c, // 12 + + 0x01, 0x04, // 4: + // "KANAYA" struct: + + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + } + expect = map[int]testStructManyFields{ + 4: {Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true, hidden: nil, enabled: nil, inc: 0}, + 2: {Name: "ROSE", Value: 12, Factor: 0.00390625, Enabled: false, hidden: nil, enabled: nil, inc: 0}, + } + expectConsumed = 106 + ) + + // execute + var actual map[int]testStructManyFields + consumed, err := Dec(input, &actual) + + // assert + if !assert.NoError(err) { + return + } + + assert.Equal(expectConsumed, consumed) + assert.Equal(expect, actual) + }) + t.Run("map[int][]int", func(t *testing.T) { // setup assert := assert.New(t) @@ -2585,6 +2840,151 @@ func Test_Dec_Map_ValueIndirection(t *testing.T) { assert.Equal(expect, actual) }) + t.Run("map[int]*struct, all non-nil", func(t *testing.T) { + // setup + assert := assert.New(t) + var ( + input = []byte{ + 0x01, 0x68, // len=104 + + 0x01, 0x02, // 2: + // "ROSE" struct: + + 0x01, 0x2f, // struct len=47 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x00, // false + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0x70, // 0.00390625 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x04, 0x52, 0x4f, 0x53, 0x45, // "ROSE" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x0c, // 12 + + 0x01, 0x04, // 4: + // "KANAYA" struct: + + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + } + expect = map[int]*testStructManyFields{ + 4: {Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true, hidden: nil, enabled: nil, inc: 0}, + 2: {Name: "ROSE", Value: 12, Factor: 0.00390625, Enabled: false, hidden: nil, enabled: nil, inc: 0}, + } + expectConsumed = 106 + ) + + // execute + var actual map[int]*testStructManyFields + consumed, err := Dec(input, &actual) + + // assert + if !assert.NoError(err) { + return + } + + assert.Equal(expectConsumed, consumed) + assert.Equal(expect, actual) + }) + + t.Run("map[int]*struct, one nil", func(t *testing.T) { + // setup + assert := assert.New(t) + var ( + input = []byte{ + 0x01, 0x36, // len=54 + + 0x01, 0x02, // 2: + // "ROSE" struct: + + 0x01, 0x2f, // struct len=47 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x00, // false + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0x70, // 0.00390625 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x04, 0x52, 0x4f, 0x53, 0x45, // "ROSE" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x0c, // 12 + + 0x01, 0x04, // 4: + // "KANAYA" struct: + + 0xa0, // nil + } + + expect = map[int]*testStructManyFields{ + 4: nil, + 2: {Name: "ROSE", Value: 12, Factor: 0.00390625, Enabled: false, hidden: nil, enabled: nil, inc: 0}, + } + + expectConsumed = 56 + ) + + // execute + var actual map[int]*testStructManyFields + consumed, err := Dec(input, &actual) + + // assert + if !assert.NoError(err) { + return + } + + assert.Equal(expectConsumed, consumed) + assert.Equal(expect, actual) + }) + + t.Run("map[int]*struct, all nil", func(t *testing.T) { + // setup + assert := assert.New(t) + var ( + input = []byte{ + 0x01, 0x06, // len=6 + + 0x01, 0x02, // 2: + 0xa0, + + 0x01, 0x04, // 4: + 0xa0, // nil + } + expect = map[int]*testStructManyFields{ + 4: nil, + 2: nil, + } + expectConsumed = 8 + ) + + // execute + var actual map[int]*testStructManyFields + consumed, err := Dec(input, &actual) + + // assert + if !assert.NoError(err) { + return + } + + assert.Equal(expectConsumed, consumed) + assert.Equal(expect, actual) + }) + t.Run("map[int]*[]int, all non-nil", func(t *testing.T) { // setup assert := assert.New(t) diff --git a/primitives.go b/primitives.go index ac63966..07f74ef 100644 --- a/primitives.go +++ b/primitives.go @@ -363,9 +363,10 @@ func decCheckedPrim(data []byte, v interface{}, ti typeInfo) (int, error) { func(t reflect.Type) bool { return t.Implements(refBinaryUnmarshalerType) }, - func(b []byte, unwrapped interface{}) (int, error) { + func(b []byte, unwrapped interface{}) (interface{}, int, error) { recv := unwrapped.(encoding.BinaryUnmarshaler) - return decBinary(b, recv) + decN, err := decBinary(b, recv) + return nil, decN, err }, )) if err != nil { @@ -389,9 +390,10 @@ func decCheckedPrim(data []byte, v interface{}, ti typeInfo) (int, error) { func(t reflect.Type) bool { return t.Implements(refTextUnmarshalerType) }, - func(b []byte, unwrapped interface{}) (int, error) { + func(b []byte, unwrapped interface{}) (interface{}, int, error) { recv := unwrapped.(encoding.TextUnmarshaler) - return decText(b, recv) + decN, err := decText(b, recv) + return nil, decN, err }, )) if err != nil { @@ -506,17 +508,19 @@ func encBool(b bool) []byte { return enc } -func decBool(data []byte) (bool, int, error) { +// returned interface{} is only there to implement decFunc and will always be +// nil +func decBool(data []byte) (bool, interface{}, int, error) { if len(data) < 1 { - return false, 0, errorDecf(0, "%s", io.ErrUnexpectedEOF).wrap(ErrMalformedData) + return false, nil, 0, errorDecf(0, "%s", io.ErrUnexpectedEOF).wrap(ErrMalformedData) } if data[0] == 0 { - return false, 1, nil + return false, nil, 1, nil } else if data[0] == 1 { - return true, 1, nil + return true, nil, 1, nil } else { - return false, 0, errorDecf(0, "not a bool value 0x00 or 0x01: %#02x", data[0]).wrap(ErrMalformedData) + return false, nil, 0, errorDecf(0, "not a bool value 0x00 or 0x01: %#02x", data[0]).wrap(ErrMalformedData) } } @@ -552,20 +556,21 @@ func encComplex[E anyComplex](v E) []byte { return enc } -func decComplex[E anyComplex](data []byte) (E, int, error) { +// return interface is just to implement decFunc and will always be nils +func decComplex[E anyComplex](data []byte) (E, interface{}, int, error) { if len(data) < 1 { - return 0.0, 0, errorDecf(0, "%s", io.ErrUnexpectedEOF).wrap(ErrMalformedData) + return 0.0, nil, 0, errorDecf(0, "%s", io.ErrUnexpectedEOF).wrap(ErrMalformedData) } // special case single-byte 0's check if data[0] == 0x00 { - return E(0.0 + 0.0i), 1, nil + return E(0.0 + 0.0i), nil, 1, nil } else if data[0] == 0x80 { // only way to reliably get a -0.0 value is by direct calculation on var // (cannot be result of consts, I tried, at least as of Go 1.19.4) var val float64 val *= -1.0 - return E(complex(val, val)), 1, nil + return E(complex(val, val)), nil, 1, nil } // do normal decoding of full-form @@ -577,9 +582,9 @@ func decComplex[E anyComplex](data []byte) (E, int, error) { var iPart float64 // get the byte count as an int - byteCount, n, err = decInt[tLen](data[offset:]) + byteCount, _, n, err = decInt[tLen](data[offset:]) if err != nil { - return E(0.0 + 0.0i), 0, err + return E(0.0 + 0.0i), nil, 0, err } offset += n @@ -593,29 +598,29 @@ func decComplex[E anyComplex](data []byte) (E, int, error) { } const errFmt = "decoded complex value byte count is %d but only %d byte%s remain%s at offset" err := errorDecf(offset, errFmt, byteCount, len(data), s, verbS).wrap(io.ErrUnexpectedEOF, ErrMalformedData) - return E(0.0 + 0.0i), 0, err + return E(0.0 + 0.0i), nil, 0, err } // clamp data to len data = data[:offset+byteCount] // real part - rPart, n, err = decFloat[float64](data[offset:]) + rPart, _, n, err = decFloat[float64](data[offset:]) if err != nil { - return E(0.0 + 0.0i), 0, errorDecf(offset, "%s", err) + return E(0.0 + 0.0i), nil, 0, errorDecf(offset, "%s", err) } offset += n // imaginary part - iPart, n, err = decFloat[float64](data[offset:]) + iPart, _, n, err = decFloat[float64](data[offset:]) if err != nil { - return E(0.0 + 0.0i), 0, errorDecf(offset, "%s", err) + return E(0.0 + 0.0i), nil, 0, errorDecf(offset, "%s", err) } offset += n var v128 complex128 = complex(rPart, iPart) - return E(v128), offset, nil + return E(v128), nil, offset, nil } func encFloat[E anyFloat](v E) []byte { @@ -729,20 +734,21 @@ func encFloat[E anyFloat](v E) []byte { return enc } -func decFloat[E anyFloat](data []byte) (E, int, error) { +// returned interface{} is just to implement decFunc; will always be nil +func decFloat[E anyFloat](data []byte) (E, interface{}, int, error) { if len(data) < 1 { - return 0.0, 0, errorDecf(0, "%s", io.ErrUnexpectedEOF).wrap(ErrMalformedData) + return 0.0, nil, 0, errorDecf(0, "%s", io.ErrUnexpectedEOF).wrap(ErrMalformedData) } byteCount := data[0] // special case single-byte 0's check if byteCount == 0 { - return E(0.0), 1, nil + return E(0.0), nil, 1, nil } else if byteCount == 0x80 { var val float64 val *= -1.0 - return E(val), 1, nil + return E(val), nil, 1, nil } // pull count and sign out of byteCount @@ -756,7 +762,7 @@ func decFloat[E anyFloat](data []byte) (E, int, error) { if len(data[1:]) < 1 { const errFmt = "count header indicates extension byte follows, but at end of data" err := errorDecf(numHeaderBytes, errFmt).wrap(io.ErrUnexpectedEOF, ErrMalformedData) - return E(0.0), 0, err + return E(0.0), nil, 0, err } data = data[1:] numHeaderBytes++ @@ -773,14 +779,14 @@ func decFloat[E anyFloat](data []byte) (E, int, error) { if negative { val *= -1.0 } - return E(val), numHeaderBytes, nil + return E(val), nil, numHeaderBytes, nil } if int(byteCount) < 2 { // the absolute minimum is 2 if not 0 const errFmt = "min data len for non-zero float is 2, but count from header specifies len of %d starting at offset" err := errorDecf(numHeaderBytes, errFmt, int(byteCount)).wrap(ErrMalformedData) - return E(0.0), 0, err + return E(0.0), nil, 0, err } if len(data) < int(byteCount) { @@ -792,7 +798,7 @@ func decFloat[E anyFloat](data []byte) (E, int, error) { } const errFmt = "decoded float byte count is %d but only %d byte%s remain%s at offset" err := errorDecf(numHeaderBytes, errFmt, byteCount, len(data), s, verbS).wrap(io.ErrUnexpectedEOF, ErrMalformedData) - return E(0.0), 0, err + return E(0.0), nil, 0, err } floatData := data[:byteCount] @@ -847,7 +853,7 @@ func decFloat[E anyFloat](data []byte) (E, int, error) { fVal := math.Float64frombits(iVal) - return E(fVal), int(byteCount) + numHeaderBytes, nil + return E(fVal), nil, int(byteCount) + numHeaderBytes, nil } func encInt[E integral](v E) []byte { @@ -900,15 +906,18 @@ func encInt[E integral](v E) []byte { // assumes that first byte specifies a non-nil integer whose L field gives // number of bytes to decode after all count header bytes and interprets it as // such. does not do further checks on count header. -func decInt[E integral](data []byte) (E, int, error) { +// +// returned interface{} is included only to implement decFunc and will always be +// nil +func decInt[E integral](data []byte) (E, interface{}, int, error) { if len(data) < 1 { - return 0, 0, errorDecf(0, "%s", io.ErrUnexpectedEOF).wrap(ErrMalformedData) + return 0, nil, 0, errorDecf(0, "%s", io.ErrUnexpectedEOF).wrap(ErrMalformedData) } byteCount := data[0] if byteCount == 0 { - return 0, 1, nil + return 0, nil, 1, nil } // pull count and sign out of byteCount @@ -922,7 +931,7 @@ func decInt[E integral](data []byte) (E, int, error) { if len(data[1:]) < 1 { const errFmt = "count header indicates extension byte follows, but at end of data" err := errorDecf(numHeaderBytes, errFmt).wrap(io.ErrUnexpectedEOF, ErrMalformedData) - return 0, 0, err + return 0, nil, 0, err } data = data[1:] numHeaderBytes++ @@ -942,7 +951,7 @@ func decInt[E integral](data []byte) (E, int, error) { } const errFmt = "decoded int byte count is %d but only %d byte%s remain%s at offset" err := errorDecf(numHeaderBytes, errFmt, byteCount, len(data), s, verbS).wrap(io.ErrUnexpectedEOF, ErrMalformedData) - return 0, 0, err + return 0, nil, 0, err } intData := data[:byteCount] @@ -973,7 +982,7 @@ func decInt[E integral](data []byte) (E, int, error) { iVal |= (uint64(intData[6]) << 8) iVal |= (uint64(intData[7])) - return E(iVal), int(byteCount) + numHeaderBytes, nil + return E(iVal), nil, int(byteCount) + numHeaderBytes, nil } func encString(s string) []byte { @@ -998,19 +1007,21 @@ func encString(s string) []byte { } // decString decodes a string of any version. Assumes header is not nil. -func decString(data []byte) (string, int, error) { +// +// returned interface{} is only to implement decFunc and will always be nil +func decString(data []byte) (string, interface{}, int, error) { if len(data) < 1 { - return "", 0, errorDecf(0, "%s", io.ErrUnexpectedEOF).wrap(ErrMalformedData) + return "", nil, 0, errorDecf(0, "%s", io.ErrUnexpectedEOF).wrap(ErrMalformedData) } // special case; 0x00 is the empty string in all variants if data[0] == 0 { - return "", 1, nil + return "", nil, 1, nil } hdr, _, err := decCountHeader(data) if err != nil { - return "", 0, err + return "", nil, 0, err } // compatibility with older format @@ -1021,18 +1032,19 @@ func decString(data []byte) (string, int, error) { return decStringV1(data) } -func decStringV1(data []byte) (string, int, error) { +// returned interface{} is only to implement decFunc and will always be nil +func decStringV1(data []byte) (string, interface{}, int, error) { if len(data) < 1 { - return "", 0, errorDecf(0, "%s", io.ErrUnexpectedEOF).wrap(ErrMalformedData) + return "", nil, 0, errorDecf(0, "%s", io.ErrUnexpectedEOF).wrap(ErrMalformedData) } - strLength, countLen, err := decInt[tLen](data) + strLength, _, countLen, err := decInt[tLen](data) if err != nil { - return "", 0, errorDecf(0, "decode string byte count: %s", err) + return "", nil, 0, errorDecf(0, "decode string byte count: %s", err) } data = data[countLen:] if strLength < 0 { - return "", 0, errorDecf(countLen, "string byte count < 0").wrap(ErrMalformedData) + return "", nil, 0, errorDecf(countLen, "string byte count < 0").wrap(ErrMalformedData) } if len(data) < strLength { @@ -1044,7 +1056,7 @@ func decStringV1(data []byte) (string, int, error) { } const errFmt = "decoded string byte count is %d but only %d byte%s remain%s at offset" err := errorDecf(countLen, errFmt, strLength, len(data), s, verbS).wrap(io.ErrUnexpectedEOF, ErrMalformedData) - return "", 0, err + return "", nil, 0, err } // clamp it data = data[:strLength] @@ -1055,7 +1067,7 @@ func decStringV1(data []byte) (string, int, error) { for readBytes-countLen < strLength { ch, charBytesRead, err := decUTF8Codepoint(data) if err != nil { - return "", 0, errorDecf(readBytes, "%s", err) + return "", nil, 0, errorDecf(readBytes, "%s", err) } sb.WriteRune(ch) @@ -1063,21 +1075,22 @@ func decStringV1(data []byte) (string, int, error) { data = data[charBytesRead:] } - return sb.String(), readBytes, nil + return sb.String(), nil, readBytes, nil } -func decStringV0(data []byte) (string, int, error) { +// returned interface{} is only to implement decFunc and will always be nil +func decStringV0(data []byte) (string, interface{}, int, error) { if len(data) < 1 { - return "", 0, errorDecf(0, "%s", io.ErrUnexpectedEOF).wrap(ErrMalformedData) + return "", nil, 0, errorDecf(0, "%s", io.ErrUnexpectedEOF).wrap(ErrMalformedData) } - runeCount, n, err := decInt[int](data) + runeCount, _, n, err := decInt[int](data) if err != nil { - return "", 0, errorDecf(0, "decode string rune count: %s", err) + return "", nil, 0, errorDecf(0, "decode string rune count: %s", err) } data = data[n:] if runeCount < 0 { - return "", 0, errorDecf(0, "string rune count < 0").wrap(ErrMalformedData) + return "", nil, 0, errorDecf(0, "string rune count < 0").wrap(ErrMalformedData) } readBytes := n @@ -1087,7 +1100,7 @@ func decStringV0(data []byte) (string, int, error) { for i := 0; i < runeCount; i++ { ch, charBytesRead, err := decUTF8Codepoint(data) if err != nil { - return "", 0, errorDecf(readBytes, "%s", err) + return "", nil, 0, errorDecf(readBytes, "%s", err) } sb.WriteRune(ch) @@ -1095,7 +1108,7 @@ func decStringV0(data []byte) (string, int, error) { data = data[charBytesRead:] } - return sb.String(), readBytes, nil + return sb.String(), nil, readBytes, nil } func decUTF8Codepoint(data []byte) (rune, int, error) { @@ -1131,7 +1144,7 @@ func decText(data []byte, t encoding.TextUnmarshaler) (int, error) { var textData string var err error - textData, readBytes, err = decString(data) + textData, _, readBytes, err = decString(data) if err != nil { return readBytes, errorDecf(0, "decode text: %s", err).wrap(ErrMalformedData) } @@ -1164,7 +1177,7 @@ func decBinary(data []byte, b encoding.BinaryUnmarshaler) (int, error) { var byteLen int var err error - byteLen, readBytes, err = decInt[tLen](data) + byteLen, _, readBytes, err = decInt[tLen](data) if err != nil { return 0, errorDecf(0, "decode byte count: %s", err) } diff --git a/primitives_test.go b/primitives_test.go index 0df72b0..3d45a4a 100644 --- a/primitives_test.go +++ b/primitives_test.go @@ -95,7 +95,7 @@ func Test_decBool(t *testing.T) { t.Run(tc.name, func(t *testing.T) { assert := assert.New(t) - actualValue, actualRead, err := decBool(tc.input) + actualValue, _, actualRead, err := decBool(tc.input) if tc.expectError { assert.Error(err) return @@ -224,7 +224,7 @@ func Test_decInt(t *testing.T) { t.Run(tc.name, func(t *testing.T) { assert := assert.New(t) - actualValue, actualRead, err := decInt[int](tc.input) + actualValue, _, actualRead, err := decInt[int](tc.input) if tc.expectError { assert.Error(err) return @@ -403,7 +403,7 @@ func Test_decFloat(t *testing.T) { assert := assert.New(t) expectBits := math.Float64bits(tc.expect) - actual, actualRead, err := decFloat[float64](tc.input) + actual, _, actualRead, err := decFloat[float64](tc.input) if tc.expectErr { assert.Error(err) return @@ -619,7 +619,7 @@ func Test_decComplex(t *testing.T) { expectRBits := math.Float64bits(real(tc.expect)) expectIBits := math.Float64bits(imag(tc.expect)) - actual, actualRead, err := decComplex[complex128](tc.input) + actual, _, actualRead, err := decComplex[complex128](tc.input) if tc.expectErr { assert.Error(err) return @@ -715,7 +715,7 @@ func Test_decStringV0(t *testing.T) { t.Run(tc.name, func(t *testing.T) { assert := assert.New(t) - actualValue, actualRead, err := decStringV0(tc.input) + actualValue, _, actualRead, err := decStringV0(tc.input) if tc.expectError { if !assert.Error(err) { return @@ -775,7 +775,7 @@ func Test_decStringV1(t *testing.T) { t.Run(tc.name, func(t *testing.T) { assert := assert.New(t) - actualValue, actualRead, err := decStringV1(tc.input) + actualValue, _, actualRead, err := decStringV1(tc.input) if tc.expectError { if !assert.Error(err) { return diff --git a/rezi.go b/rezi.go index 5d89c85..a1e0f1e 100644 --- a/rezi.go +++ b/rezi.go @@ -1,5 +1,5 @@ // Package rezi provides the ability to encode and decode data in Rarefied -// Encoding (Compressible) Interchange format. It allows basic Go types and +// Encoding (Compressible) Interchange format. It allows Go types and // user-defined types to be easily read from and written to byte slices, with // customization possible by implementing encoding.BinaryUnmarshaler and // encoding.BinaryMarshaler on a type, or alternatively by implementing @@ -138,13 +138,18 @@ // not have any concept of two different pointer variables pointing to the same // data. // -// Besides the above listed types, all types whose underlying type is a -// supported type are themselves supported as well. For example, time.Duration -// has an underlying type of int64, and is therefore supported in REZI. This -// does not apply to marshaler implementors; a type whose underlying type is -// only supported in REZI via implementation of one of the marshaler or -// unmarshaler interfaces must itself implement that interface -// in order to be fully supported. +// All non-struct types whose underlying type is a supported type are themselves +// supported as well. For example, time.Duration has an underlying type of +// int64, and is therefore supported in REZI. This does not apply to marshaler +// implementors; a type whose underlying type is only supported in REZI via +// implementation of one of the marshaler or unmarshaler interfaces must itself +// implement that interface in order to be fully supported. +// +// Struct types are supported even if they do not implement text or binary +// marshaling functions, provided all of their exported fields are of a +// supported type. Both decoding and encoding ignore all unexported fields. If a +// field is not present in the given bytes during decoding, its original value +// is left intact, even if it is exported. // // # Binary Data Format // @@ -410,6 +415,25 @@ // encoding; if a type implements both, it will be encoded as a BinaryMarshaler, // not a TextMarshaler. // +// Struct Values +// +// Layout: +// +// [ INFO ] [ INT VALUE ] [ FIELD 1 ] [ VALUE 1 ] ... [ FIELD N ] [ VALUE N ] +// <-------COUNT--------> <---------------------VALUES----------------------> +// 1..9 bytes COUNT bytes +// +// Structs that do not implement binary marshaling or text marshaling funcitons +// are encoded as a count of all bytes that make up the entire struct, followed +// by pairs of the names and associated values for each exported field of the +// struct. Each pair consists of the case-sensitive name of the field encoded as +// a string, followed immediately by the encoded value of that field. There is +// no special delimiter between name-value pairs or between the name and value +// in a pair; where one ends, the next one begins. +// +// The encoded names are placed in a consistent order; encoding the same struct +// will result in the same encoding. +// // Slice Values // // Layout: @@ -531,7 +555,9 @@ type ( tLen = int tNilLevel = int - decFunc[E any] func([]byte) (E, int, error) + // the interface{} in decFunc is any additional info needed for further + // stages of decode, nil in all cases except for struct decoding. + decFunc[E any] func([]byte) (E, interface{}, int, error) encFunc[E any] func(E) ([]byte, error) ) @@ -588,6 +614,8 @@ func Enc(v interface{}) (data []byte, err error) { return encCheckedMap(v, info) } else if info.Main == mtSlice || info.Main == mtArray { return encCheckedSlice(v, info) + } else if info.Main == mtStruct { + return encCheckedStruct(v, info) } else { panic("no possible encoding") } @@ -632,11 +660,11 @@ func MustDec(data []byte, v interface{}) int { // the data itself (including there being fewer bytes than necessary to decode // the value). func Dec(data []byte, v interface{}) (n int, err error) { - defer func() { - if r := recover(); r != nil { - err = errorf("%v", r) - } - }() + // defer func() { + // if r := recover(); r != nil { + // err = errorf("%v", r) + // } + // }() info, err := canDecode(v) if err != nil { @@ -649,6 +677,8 @@ func Dec(data []byte, v interface{}) (n int, err error) { return decCheckedMap(data, v, info) } else if info.Main == mtSlice || info.Main == mtArray { return decCheckedSlice(data, v, info) + } else if info.Main == mtStruct { + return decCheckedStruct(data, v, info) } else { panic("no possible decoding") } @@ -703,8 +733,10 @@ func decWithNilCheck[E any](data []byte, v interface{}, ti typeInfo, decFn decFu countHeaderBytes := n effectiveExtraIndirs := hdr.ExtraNilIndirections() + var extraInfo interface{} + if !hdr.IsNil() { - decoded, n, err = decFn(data) + decoded, extraInfo, n, err = decFn(data) if err != nil { return decoded, n, errorDecf(countHeaderBytes, "%s", err) } @@ -716,6 +748,13 @@ func decWithNilCheck[E any](data []byte, v interface{}, ti typeInfo, decFn decFu assignTarget := reflect.ValueOf(v) // assignTarget is a **string but we want a *string + // if it's a struct, we must get the original value, if one exists, in order + // to preserve the original member values + var origStructVal reflect.Value + if ti.Main == mtStruct { + origStructVal = unwrapOriginalStructValue(assignTarget) + } + for i := 0; i < ti.Indir && i < effectiveExtraIndirs; i++ { // *double indirection ALL THE WAY~* // *acrosssss the sky* @@ -732,6 +771,11 @@ func decWithNilCheck[E any](data []byte, v interface{}, ti typeInfo, decFn decFu if ti.Underlying { refDecoded = refDecoded.Convert(assignTarget.Type().Elem()) } + + if ti.Main == mtStruct && origStructVal.IsValid() { + refDecoded = setStructMembers(origStructVal, refDecoded, extraInfo.([]fieldInfo)) + } + assignTarget.Elem().Set(refDecoded) } else { zeroVal := reflect.Zero(assignTarget.Elem().Type()) @@ -742,8 +786,10 @@ func decWithNilCheck[E any](data []byte, v interface{}, ti typeInfo, decFn decFu return decoded, n, nil } -func fn_DecToWrappedReceiver(wrapped interface{}, ti typeInfo, assertFn func(reflect.Type) bool, decToUnwrappedFn func([]byte, interface{}) (int, error)) decFunc[interface{}] { - return func(data []byte) (interface{}, int, error) { +// decToUnwrappedFn takes the encoded bytes and an interface to decode to and +// returns any extra data (may be nil), bytes consumed, and error status. +func fn_DecToWrappedReceiver(wrapped interface{}, ti typeInfo, assertFn func(reflect.Type) bool, decToUnwrappedFn func([]byte, interface{}) (interface{}, int, error)) decFunc[interface{}] { + return func(data []byte) (interface{}, interface{}, int, error) { // v is *(...*)T, ret-val of decFn (this lambda) is T. receiverType := reflect.TypeOf(wrapped) @@ -767,7 +813,7 @@ func fn_DecToWrappedReceiver(wrapped interface{}, ti typeInfo, assertFn func(ref if receiverType.Elem().Kind() == reflect.Func { // if we have been given a *function* pointer, reject it, we // cannot do this. - return nil, 0, errorDecf(0, "function pointer type receiver is not supported").wrap(ErrInvalidType) + return nil, nil, 0, errorDecf(0, "function pointer type receiver is not supported").wrap(ErrInvalidType) } // receiverType is *T receiverValue = reflect.New(receiverType.Elem()) @@ -779,10 +825,10 @@ func fn_DecToWrappedReceiver(wrapped interface{}, ti typeInfo, assertFn func(ref var decoded interface{} receiver := receiverValue.Interface() - decConsumed, decErr := decToUnwrappedFn(data, receiver) + extraInfo, decConsumed, decErr := decToUnwrappedFn(data, receiver) if decErr != nil { - return nil, decConsumed, decErr + return nil, extraInfo, decConsumed, decErr } if receiverType.Kind() == reflect.Pointer { @@ -791,7 +837,7 @@ func fn_DecToWrappedReceiver(wrapped interface{}, ti typeInfo, assertFn func(ref decoded = receiver } - return decoded, decConsumed, decErr + return decoded, extraInfo, decConsumed, decErr } } @@ -983,7 +1029,7 @@ func (hdr *countHeader) UnmarshalBinary(data []byte) error { // all extension bytes processed, now decode any indirection level int if // present if infoByte&infoBitsIndir != 0 { - extraIndirs, n, err := decInt[tNilLevel](data[decoded.DecodedCount:]) + extraIndirs, _, n, err := decInt[tNilLevel](data[decoded.DecodedCount:]) if err != nil { return errorDecf(decoded.DecodedCount, "%s", err) } diff --git a/rezi_test.go b/rezi_test.go index 100fe6c..633dde0 100644 --- a/rezi_test.go +++ b/rezi_test.go @@ -13,7 +13,7 @@ import ( ) func Test_Enc_Errors(t *testing.T) { - type dummyType struct{} + dummyTyped := make(chan struct{}) ErrFakeMarshal := errors.New("fake marshal error") @@ -24,12 +24,12 @@ func Test_Enc_Errors(t *testing.T) { }{ { name: "unknown type - Error", - input: dummyType{}, + input: dummyTyped, expectErr: Error, }, { name: "unknown type - ErrInvalidType", - input: dummyType{}, + input: dummyTyped, expectErr: ErrInvalidType, }, { @@ -61,7 +61,7 @@ func Test_Enc_Errors(t *testing.T) { } func Test_Dec_Errors(t *testing.T) { - type dummyType struct{} + dummyTyped := make(chan struct{}) testCases := []struct { name string @@ -91,12 +91,12 @@ func Test_Dec_Errors(t *testing.T) { }, { name: "receiver is unsupported type - Error", - recv: ref(dummyType{}), + recv: ref(dummyTyped), expectErr: Error, }, { name: "receiver is unsupported type - ErrInvalidType", - recv: ref(dummyType{}), + recv: ref(dummyTyped), expectErr: ErrInvalidType, }, { diff --git a/slices.go b/slices.go index 4372a54..eba4ae9 100644 --- a/slices.go +++ b/slices.go @@ -53,7 +53,10 @@ func decCheckedSlice(data []byte, v interface{}, ti typeInfo) (int, error) { func(t reflect.Type) bool { return t.Kind() == reflect.Pointer && ((ti.Main == mtSlice && t.Elem().Kind() == reflect.Slice) || (ti.Main == mtArray && t.Elem().Kind() == reflect.Array)) }, - decSlice, + func(b []byte, i interface{}) (interface{}, int, error) { + decN, err := decSlice(b, i) + return nil, decN, err + }, )) if err != nil { return n, err @@ -68,7 +71,7 @@ func decCheckedSlice(data []byte, v interface{}, ti typeInfo) (int, error) { func decSlice(data []byte, v interface{}) (int, error) { var totalConsumed int - toConsume, n, err := decInt[tLen](data) + toConsume, _, n, err := decInt[tLen](data) if err != nil { return 0, errorDecf(0, "decode byte count: %s", err) } diff --git a/slices_test.go b/slices_test.go index 4c298de..3746042 100644 --- a/slices_test.go +++ b/slices_test.go @@ -3,6 +3,7 @@ package rezi import ( "encoding/asn1" "math" + "sync" "testing" "github.com/stretchr/testify/assert" @@ -263,6 +264,62 @@ func Test_Enc_Slice_NoIndirection(t *testing.T) { assert.Equal(expect, actual) }) + t.Run("[]struct", func(t *testing.T) { + // setup + assert := assert.New(t) + var ( + input = []testStructManyFields{ + {Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true, hidden: make(chan int, 3), enabled: &sync.Mutex{}, inc: 48}, + {Name: "ROSE", Value: 12, Factor: 0.00390625, Enabled: false, hidden: make(chan int), enabled: nil, inc: 12}, + } + + expect = []byte{ + 0x01, 0x64, // len=100 + + // "KANAYA" struct: + + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + + // "ROSE" struct: + + 0x01, 0x2f, // struct len=47 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x00, // false + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0x70, // 0.00390625 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x04, 0x52, 0x4f, 0x53, 0x45, // "ROSE" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x0c, // 12 + } + ) + + // execute + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expect, actual) + }) + t.Run("[]map[string]bool", func(t *testing.T) { // setup assert := assert.New(t) @@ -1033,6 +1090,129 @@ func Test_Enc_Slice_ValueIndirection(t *testing.T) { assert.Equal(expect, actual) }) + t.Run("[]*struct, all non-nil", func(t *testing.T) { + // setup + assert := assert.New(t) + var ( + input = []*testStructManyFields{ + {Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true, hidden: make(chan int, 3), enabled: &sync.Mutex{}, inc: 48}, + {Name: "ROSE", Value: 12, Factor: 0.00390625, Enabled: false, hidden: make(chan int), enabled: nil, inc: 12}, + } + + expect = []byte{ + 0x01, 0x64, // len=100 + + // "KANAYA" struct: + + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + + // "ROSE" struct: + + 0x01, 0x2f, // struct len=47 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x00, // false + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0x70, // 0.00390625 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x04, 0x52, 0x4f, 0x53, 0x45, // "ROSE" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x0c, // 12 + } + ) + + // execute + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expect, actual) + }) + + t.Run("[]*struct, one nil", func(t *testing.T) { + // setup + assert := assert.New(t) + var ( + input = []*testStructManyFields{ + {Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true, hidden: make(chan int, 3), enabled: &sync.Mutex{}, inc: 48}, nil, + } + + expect = []byte{ + 0x01, 0x34, // len=52 + + // "KANAYA" struct: + + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + + // "ROSE" struct: + + 0xa0, // nil + } + ) + + // execute + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expect, actual) + }) + + t.Run("[]*struct, all nil", func(t *testing.T) { + // setup + assert := assert.New(t) + var ( + input = []*testStructManyFields{nil, nil} + + expect = []byte{ + 0x01, 0x02, // len=2 + + 0xa0, + 0xa0, + } + ) + + // execute + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expect, actual) + }) + t.Run("[]*map[string]bool, all non-nil", func(t *testing.T) { // setup assert := assert.New(t) @@ -1615,6 +1795,65 @@ func Test_Dec_Slice_NoIndirection(t *testing.T) { assert.Equal(expect, actual) }) + t.Run("[]struct", func(t *testing.T) { + assert := assert.New(t) + + var ( + input = []byte{ + 0x01, 0x64, // len=100 + + // "KANAYA" struct: + + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + + // "ROSE" struct: + + 0x01, 0x2f, // struct len=47 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x00, // false + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0x70, // 0.00390625 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x04, 0x52, 0x4f, 0x53, 0x45, // "ROSE" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x0c, // 12 + } + expect = []testStructManyFields{ + {Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true, hidden: nil, enabled: nil, inc: 0}, + {Name: "ROSE", Value: 12, Factor: 0.00390625, Enabled: false, hidden: nil, enabled: nil, inc: 0}, + } + expectConsumed = 102 + ) + + // execute + var actual []testStructManyFields + consumed, err := Dec(input, &actual) + + // assert + if !assert.NoError(err) { + return + } + + assert.Equal(expectConsumed, consumed) + assert.Equal(expect, actual) + }) + t.Run("[]map[string]bool", func(t *testing.T) { // setup assert := assert.New(t) @@ -2485,6 +2724,139 @@ func Test_Dec_Slice_ValueIndirection(t *testing.T) { assert.Equal(expect, actual) }) + t.Run("[]*struct, all non-nil", func(t *testing.T) { + // setup + assert := assert.New(t) + + var ( + input = []byte{ + 0x01, 0x64, // len=100 + + // "KANAYA" struct: + + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + + // "ROSE" struct: + + 0x01, 0x2f, // struct len=47 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x00, // false + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0x70, // 0.00390625 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x04, 0x52, 0x4f, 0x53, 0x45, // "ROSE" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x0c, // 12 + } + expect = []*testStructManyFields{ + {Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true, hidden: nil, enabled: nil, inc: 0}, + {Name: "ROSE", Value: 12, Factor: 0.00390625, Enabled: false, hidden: nil, enabled: nil, inc: 0}, + } + expectConsumed = 102 + ) + + // execute + var actual []*testStructManyFields + consumed, err := Dec(input, &actual) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expectConsumed, consumed) + assert.Equal(expect, actual) + }) + + t.Run("[]*struct, one nil", func(t *testing.T) { + // setup + assert := assert.New(t) + + var ( + input = []byte{ + 0x01, 0x34, // len=52 + + // "KANAYA" struct: + + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + + // "ROSE" struct: + + 0xa0, // nil + } + expect = []*testStructManyFields{ + {Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true, hidden: nil, enabled: nil, inc: 0}, + nil, + } + expectConsumed = 54 + ) + + // execute + var actual []*testStructManyFields + consumed, err := Dec(input, &actual) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expectConsumed, consumed) + assert.Equal(expect, actual) + }) + + t.Run("[]*struct, all nil", func(t *testing.T) { + // setup + assert := assert.New(t) + + var ( + input = []byte{ + 0x01, 0x02, // len=2 + + 0xa0, + 0xa0, + } + expect = []*testStructManyFields{nil, nil} + expectConsumed = 4 + ) + + // execute + var actual []*testStructManyFields + consumed, err := Dec(input, &actual) + if !assert.NoError(err) { + return + } + + // assert + assert.Equal(expectConsumed, consumed) + assert.Equal(expect, actual) + }) + t.Run("[]*map[string]bool, all non-nil", func(t *testing.T) { // setup assert := assert.New(t) diff --git a/streams.go b/streams.go index 021d36d..b613dd6 100644 --- a/streams.go +++ b/streams.go @@ -501,7 +501,7 @@ func (r *Reader) loadDecodeableBytes(info typeInfo) ([]byte, error) { return decodable, errorDecf(totalRead, "%s", err) } - count, n, err := decInt[int](buf) + count, _, n, err := decInt[int](buf) // do not preserve this error, it will never be io.EOF. if err != nil { return decodable, errorDecf(totalRead, "header byte-count int: %s", err) @@ -524,7 +524,7 @@ func (r *Reader) loadDecodeableBytes(info typeInfo) ([]byte, error) { } // okay, we have complete header and int bytes, decode to int type - count, n, err := decInt[int](decodable) + count, _, n, err := decInt[int](decodable) // do not preserve this error, it will never be io.EOF. if err != nil { return decodable, errorDecf(0, "count header: %s", err) @@ -579,7 +579,7 @@ func (r *Reader) loadV0StringBytes(hdr countHeader, hdrBytes []byte) ([]byte, er } // okay, we have complete header and int bytes, decode to int type - runeCount, n, err := decInt[int](loaded) + runeCount, _, n, err := decInt[int](loaded) // do not preserve this error, it will never be io.EOF. if err != nil { return loaded, errorDecf(0, "count header: %s", err) diff --git a/streams_test.go b/streams_test.go index 1af0706..babddf9 100644 --- a/streams_test.go +++ b/streams_test.go @@ -134,6 +134,7 @@ func Test_Writer_Enc(t *testing.T) { intData := 413 arrData := [3]string{"VRISKA", "TEREZI"} textData := testText{name: "VRISKA", value: 8, enabled: true} + structData := testStructManyFields{Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true} expect = []byte{ 0x41, 0x82, 0x06, 0x4e, 0x45, 0x50, 0x45, 0x54, 0x41, 0x04, 0xc0, 0x70, 0x00, 0x32, @@ -146,6 +147,16 @@ func Test_Writer_Enc(t *testing.T) { 0x00, 0x41, 0x82, 0x0d, 0x38, 0x2c, 0x74, 0x72, 0x75, 0x65, 0x2c, 0x56, 0x52, 0x49, 0x53, 0x4b, 0x41, + + 0x01, 0x31, // struct len=49 + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 } err = w.Enc(strData) @@ -172,6 +183,10 @@ func Test_Writer_Enc(t *testing.T) { if !assert.NoError(err, "error writing sixth time") { return } + err = w.Enc(structData) + if !assert.NoError(err, "error writing seventh time") { + return + } w.Flush() actual := buf.Bytes() @@ -538,6 +553,24 @@ func Test_Reader_Dec_sequential(t *testing.T) { var dest13Text testText expect13Text := testText{name: "VRISKA", value: 8, enabled: true} + input = append(input, + 0x01, 0x31, // struct len=49 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x02, 0x3f, 0xd0, // 0.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + ) + var dest14Struct testStructManyFields + expect14Struct := testStructManyFields{Name: "KANAYA", Value: 8, Factor: 0.25, Enabled: true} + r, err := NewReader(bytes.NewReader(input), nil) if !assert.NoError(err, "creating Reader returned error") { return @@ -620,6 +653,12 @@ func Test_Reader_Dec_sequential(t *testing.T) { return } assert.Equal(expect13Text, dest13Text, "dest13Text mismatch") + + err = r.Dec(&dest14Struct) + if !assert.NoError(err) { + return + } + assert.Equal(expect14Struct, dest14Struct, "dest14Struct mismatch") } func Test_Reader_Dec_int(t *testing.T) { @@ -1429,6 +1468,156 @@ func Test_Reader_Dec_binary(t *testing.T) { }) } +func Test_Reader_Dec_struct(t *testing.T) { + testCases := []struct { + name string + input []byte + expect testStructMultiMember + expectErr bool + expectOff int + }{ + { + name: "empty values", + input: []byte{ + 0x01, 0x11, // len=17 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x00, // "" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x00, // 0 + }, + expect: testStructMultiMember{}, + expectOff: 19, + }, + { + name: "filled values", + input: []byte{ + 0x01, 0x1a, // len=26 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + }, + expect: testStructMultiMember{Name: "KANAYA", Value: 8}, + expectOff: 28, + }, + { + name: "empty values x2", + input: []byte{ + 0x01, 0x11, // len=17 + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x00, // "" + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x00, // 0 + + 0x01, 0x11, // len=17 + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x00, // "" + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x00, // 0 + }, + expect: testStructMultiMember{}, + expectOff: 19, + }, + { + name: "filled values x2", + input: []byte{ + 0x01, 0x1a, // len=26 + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + + 0x01, 0x1a, // len=26 + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4b, 0x41, 0x4e, 0x41, 0x59, 0x41, // "KANAYA" + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x08, // 8 + }, + expect: testStructMultiMember{Name: "KANAYA", Value: 8}, + expectOff: 28, + }, + { + // error - invalid (nil) count + name: "error - invalid indir count int", + input: []byte{0x70, 0x00, 0x20}, + expectErr: true, + expectOff: 3, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert := assert.New(t) + + r, err := NewReader(bytes.NewReader(tc.input), nil) + if !assert.NoError(err, "creating Reader returned error") { + return + } + + var dest testStructMultiMember + err = r.Dec(&dest) + if tc.expectErr { + assert.Error(err, "error not returned") + assert.Equal(tc.expectOff, r.offset, "offset mismatch") + return + } + if !assert.NoError(err) { + return + } + + assert.Equal(tc.expect, dest, "dest not expected value") + assert.Equal(tc.expectOff, r.offset, "offset mismatch") + }) + } + + t.Run("nil value - single indir", func(t *testing.T) { + assert := assert.New(t) + input := []byte{0x20} + expect := nilRef[testStructMultiMember]() + expectOff := 1 + + r, err := NewReader(bytes.NewReader(input), nil) + if !assert.NoError(err, "creating Reader returned error") { + return + } + + var dest *testStructMultiMember + err = r.Dec(&dest) + if !assert.NoError(err) { + return + } + + assert.Equal(expect, dest, "dest not expected value") + assert.Equal(expectOff, r.offset, "offset mismatch") + }) + + t.Run("nil value - multi indir", func(t *testing.T) { + assert := assert.New(t) + input := []byte{0x30, 0x01, 0x01} + expectPtr := nilRef[testStructMultiMember]() + expect := &expectPtr + expectOff := 3 + + r, err := NewReader(bytes.NewReader(input), nil) + if !assert.NoError(err, "creating Reader returned error") { + return + } + + var dest **testStructMultiMember + err = r.Dec(&dest) + if !assert.NoError(err) { + return + } + + assert.Equal(expect, dest, "dest not expected value") + assert.Equal(expectOff, r.offset, "offset mismatch") + }) +} + func Test_Reader_Dec_text(t *testing.T) { testCases := []struct { name string diff --git a/structs.go b/structs.go new file mode 100644 index 0000000..46cb87b --- /dev/null +++ b/structs.go @@ -0,0 +1,199 @@ +package rezi + +import ( + "io" + "reflect" +) + +// encCheckedStruct encodes a compatible struct as a REZI . +func encCheckedStruct(v interface{}, ti typeInfo) ([]byte, error) { + if ti.Main != mtStruct { + panic("not a struct type") + } + + return encWithNilCheck(v, ti, func(val interface{}) ([]byte, error) { + return encStruct(val, ti) + }, reflect.Value.Interface) +} + +func encStruct(v interface{}, ti typeInfo) ([]byte, error) { + refVal := reflect.ValueOf(v) + + enc := make([]byte, 0) + + for _, fi := range ti.Fields.ByOrder { + v := refVal.Field(fi.Index) + + fNameData, err := Enc(fi.Name) + if err != nil { + msgTypeName := reflect.ValueOf(v).Type().Name() + if msgTypeName == "" { + msgTypeName = "(anonymous type)" + } + return nil, errorf("%s.%s field name: %s", msgTypeName, fi.Name, err) + } + fValData, err := Enc(v.Interface()) + if err != nil { + msgTypeName := reflect.ValueOf(v).Type().Name() + if msgTypeName == "" { + msgTypeName = "(anonymous type)" + } + return nil, errorf("%s.%s: %v", msgTypeName, fi.Name, err) + } + + enc = append(enc, fNameData...) + enc = append(enc, fValData...) + } + + enc = append(encInt(tLen(len(enc))), enc...) + return enc, nil +} + +// decCheckedStruct decodes a REZI bytes representation of a struct into a +// compatible struct type. +func decCheckedStruct(data []byte, v interface{}, ti typeInfo) (int, error) { + if ti.Main != mtStruct { + panic("not a struct type") + } + + var extraInfo []fieldInfo + st, n, err := decWithNilCheck(data, v, ti, fn_DecToWrappedReceiver(v, ti, + func(t reflect.Type) bool { + return t.Kind() == reflect.Pointer && t.Elem().Kind() == reflect.Struct + }, + func(data []byte, v interface{}) (interface{}, int, error) { + fi, n, err := decStruct(data, v, ti) + extraInfo = fi + return fi, n, err + }, + )) + if err != nil { + return n, err + } + if ti.Indir == 0 { + refReceiver := reflect.ValueOf(v) + + // if it's a struct, we must get the original value, if one exists, in order + // to preserve the original member values + var origStructVal reflect.Value + if ti.Main == mtStruct { + origStructVal = unwrapOriginalStructValue(refReceiver) + } + + refSt := reflect.ValueOf(st) + + if ti.Main == mtStruct && origStructVal.IsValid() { + refSt = setStructMembers(origStructVal, refSt, extraInfo) + } + + refReceiver.Elem().Set(refSt) + } + return n, err +} + +// the fieldInfo is the successfully decoded fields. +func decStruct(data []byte, v interface{}, ti typeInfo) ([]fieldInfo, int, error) { + var decFields []fieldInfo + var totalConsumed int + + refVal := reflect.ValueOf(v) + refStructType := refVal.Type().Elem() + msgTypeName := refStructType.Name() + if msgTypeName == "" { + msgTypeName = "(anonymous type)" + } + + toConsume, _, n, err := decInt[tLen](data) + if err != nil { + return decFields, 0, errorDecf(0, "decode %s byte count: %s", msgTypeName, err) + } + data = data[n:] + totalConsumed += n + + if toConsume == 0 { + // initialize to an empty struct + emptyStruct := reflect.New(refStructType) + + // set it to the value + refVal.Elem().Set(emptyStruct.Elem()) + return decFields, totalConsumed, nil + } + + if len(data) < toConsume { + s := "s" + verbS := "" + if len(data) == 1 { + s = "" + verbS = "s" + } + const errFmt = "decoded %s byte count is %d but only %d byte%s remain%s in data at offset" + err := errorDecf(totalConsumed, errFmt, msgTypeName, toConsume, len(data), s, verbS).wrap(io.ErrUnexpectedEOF, ErrMalformedData) + return decFields, totalConsumed, err + } + + // clamp values we are allowed to read so we don't try to read other data + data = data[:toConsume] + + target := refVal.Elem() + var i int + for i < toConsume { + // get field name + var fNameVal string + n, err = Dec(data, &fNameVal) + if err != nil { + return decFields, totalConsumed, errorDecf(totalConsumed, "decode %s field name: %s", msgTypeName, err) + } + totalConsumed += n + i += n + data = data[n:] + + // get field info from name + fi, ok := ti.Fields.ByName[fNameVal] + if !ok { + return decFields, totalConsumed, errorDecf(totalConsumed, "field name .%s does not exist in decoded-to %s", fNameVal, msgTypeName).wrap(ErrMalformedData, ErrInvalidType) + } + fieldPtr := target.Field(fi.Index).Addr() + n, err = Dec(data, fieldPtr.Interface()) + if err != nil { + return decFields, totalConsumed, errorDecf(totalConsumed, "%s.%s: %v", msgTypeName, fi.Name, err) + } + totalConsumed += n + i += n + data = data[n:] + decFields = append(decFields, fi) + } + + return decFields, totalConsumed, nil +} + +func setStructMembers(initial, decoded reflect.Value, decodedFields []fieldInfo) reflect.Value { + newVal := reflect.New(initial.Type()) + newVal.Elem().Set(initial) + + for _, fi := range decodedFields { + destPtr := newVal.Elem().Field(fi.Index).Addr() + fieldVal := decoded.Field(fi.Index) + destPtr.Elem().Set(fieldVal) + } + + return newVal.Elem() +} + +// this will return nil if v does not end up in a struct value after +// dereferences are made +func unwrapOriginalStructValue(refVal reflect.Value) reflect.Value { + // TODO: move all this to type analysis + + // the user may have passed in a ptr-ptr-to, make shore we get actual + // target + for refVal.Kind() == reflect.Pointer && !refVal.IsNil() { + refVal = refVal.Elem() + } + + // only pick up orig value if we ended up at a struct type + if refVal.Kind() == reflect.Struct { + return refVal + } + + return reflect.Value{} +} diff --git a/structs_test.go b/structs_test.go new file mode 100644 index 0000000..0987ca2 --- /dev/null +++ b/structs_test.go @@ -0,0 +1,546 @@ +package rezi + +// putting this in rezi_test and not rezi because we need to create a public +// type in it and we don't want to pollute the rezi package with it. + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/assert" +) + +type testStructEmpty struct{} + +type testStructOneMember struct { + Value int +} + +type testStructMultiMember struct { + Value int + Name string +} + +type testStructWithUnexported struct { + Value int + unexported float32 +} + +type testStructWithUnexportedCaseDistinguished struct { + Value int + value float32 +} + +type testStructOnlyUnexported struct { + value int + name string +} + +type testStructManyFields struct { + Name string + Factor float64 + Value int + Enabled bool + + hidden chan int + inc int + enabled *sync.Mutex +} + +func Test_Enc_Struct(t *testing.T) { + // we require an exported struct in order to test embedded struct fields. + // we will declare it and other struct types it is embedded in here in this + // function to avoid adding a new exported type to the rezi package. + type TestStructToEmbed struct { + Value int + } + type testStructWithEmbedded struct { + TestStructToEmbed + Name string + } + + type testStructWithEmbeddedOverlap struct { + TestStructToEmbed + Name string + Value float64 + } + + runEncTests(t, "empty struct", testStructEmpty{}, []byte{0x00}) + runEncTests(t, "one member", testStructOneMember{Value: 4}, []byte{ + 0x01, 0x0a, // len=10 + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x04, // 4 + }) + runEncTests(t, "multi member", testStructMultiMember{Value: 4, Name: "NEPETA"}, []byte{ + 0x01, 0x1a, // len=26 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4e, 0x45, 0x50, 0x45, 0x54, 0x41, // "NEPETA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x04, // 4 + }) + runEncTests(t, "with unexported", testStructWithUnexported{Value: 4, unexported: 9}, []byte{ + 0x01, 0x0a, // len=10 + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x04, // 4 + }) + runEncTests(t, "with unexported case distinguished", testStructWithUnexportedCaseDistinguished{Value: 4, value: 3.2}, []byte{ + 0x01, 0x0a, // len=10 + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x04, // 4 + }) + runEncTests(t, "only unexported", testStructOnlyUnexported{value: 3, name: "test"}, []byte{0x00}) + runEncTests(t, "many fields", testStructManyFields{ + Value: 413, + Name: "Rose Lalonde", + Enabled: true, + Factor: 8.25, + + hidden: make(chan int), + inc: 17, + enabled: &sync.Mutex{}, + }, []byte{ + 0x01, 0x39, // len=57 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x03, 0xc0, 0x20, 0x80, // 8.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x0c, 0x52, 0x6f, 0x73, 0x65, 0x20, 0x4c, 0x61, 0x6c, 0x6f, 0x6e, 0x64, 0x65, // "Rose Lalonde" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x02, 0x01, 0x9d, // 413 + }) + runEncTests(t, "with embedded", testStructWithEmbedded{ + TestStructToEmbed: TestStructToEmbed{ + Value: 4, + }, + Name: "NEPETA", + }, []byte{ + 0x01, 0x30, // len=48 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4e, 0x45, 0x50, 0x45, 0x54, 0x41, // "NEPETA" + + 0x41, 0x82, 0x11, 0x54, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x6f, 0x45, 0x6d, 0x62, 0x65, 0x64, // "TestStructToEmbed" + 0x01, 0x0a, // len=10 + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x04, // 4 + }) + runEncTests(t, "with embedded overlap", testStructWithEmbeddedOverlap{ + TestStructToEmbed: TestStructToEmbed{ + Value: 4, + }, + Value: 8.25, + Name: "NEPETA", + }, []byte{ + 0x01, 0x3c, // len=60 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4e, 0x45, 0x50, 0x45, 0x54, 0x41, // "NEPETA" + + 0x41, 0x82, 0x11, 0x54, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x6f, 0x45, 0x6d, 0x62, 0x65, 0x64, // "TestStructToEmbed" + 0x01, 0x0a, // len=10 + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x04, // 4 + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x03, 0xc0, 0x20, 0x80, // 8.25 + }) +} + +func runEncTests[E any](t *testing.T, name string, inputVal E, expect []byte) { + // TODO: too complicated, just make the test cases instead of going off to a function + // normal value test + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + input := inputVal + + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + assert.Equal(expect, actual) + }) + + // single pointer, filled + t.Run("*("+name+")", func(t *testing.T) { + assert := assert.New(t) + + input := &inputVal + + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + assert.Equal(expect, actual) + }) + + // single pointer, nil + t.Run("*("+name+"), nil", func(t *testing.T) { + assert := assert.New(t) + + var input *E + var nilExp = []byte{0xa0} + + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + assert.Equal(nilExp, actual) + }) + + // double pointer, filled + t.Run("**("+name+")", func(t *testing.T) { + assert := assert.New(t) + + inputPtr := &inputVal + input := &inputPtr + + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + assert.Equal(expect, actual) + }) + + // double pointer, nil at first level + t.Run("**("+name+"), nil at first level", func(t *testing.T) { + assert := assert.New(t) + + var inputPtr *E + var input = &inputPtr + var nilFirstLevelExp = []byte{0xb0, 0x01, 0x01} + + actual, err := Enc(input) + if !assert.NoError(err) { + return + } + + assert.Equal(nilFirstLevelExp, actual) + }) +} + +func Test_Dec_Struct(t *testing.T) { + // we require an exported struct in order to test embedded struct fields. + // we will declare it and other struct types it is embedded in here in this + // function to avoid adding a new exported type to the rezi package. + type TestStructToEmbed struct { + Value int + } + type testStructWithEmbedded struct { + TestStructToEmbed + Name string + } + + type testStructWithEmbeddedOverlap struct { + TestStructToEmbed + Name string + Value float64 + } + + runDecTests(t, "no-member struct", nil, []byte{0x00}, testStructEmpty{}, nil, 1) + + runDecTests(t, "one-member struct", nil, + []byte{ + 0x01, 0x0a, // len=10 + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x04, // 4 + }, testStructOneMember{Value: 4}, nil, 12) + + runDecTests(t, "multi-member struct", nil, + []byte{ + 0x01, 0x1a, // len=26 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4e, 0x45, 0x50, 0x45, 0x54, 0x41, // "NEPETA" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x04, // 4 + }, testStructMultiMember{Value: 4, Name: "NEPETA"}, nil, 28) + + runDecTests(t, "with unexported", nil, + []byte{ + 0x01, 0x0a, // len=10 + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x04, // 4 + }, testStructWithUnexported{Value: 4, unexported: 0}, nil, 12) + + runDecTests(t, "with unexported values set", &testStructWithUnexported{unexported: 12}, + []byte{ + 0x01, 0x0a, // len=10 + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x04, // 4 + }, testStructWithUnexported{Value: 4, unexported: 12}, &testStructWithUnexported{unexported: 12}, 12) + + runDecTests(t, "with unexported case distinguished", nil, + []byte{ + 0x01, 0x0a, // len=10 + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x04, // 4 + }, testStructWithUnexportedCaseDistinguished{Value: 4, value: 0}, nil, 12) + + runDecTests(t, "only unexported", nil, []byte{0x00}, testStructOnlyUnexported{value: 0, name: ""}, nil, 1) + + runDecTests(t, "many fields", nil, + []byte{ + 0x01, 0x39, // len=57 + + 0x41, 0x82, 0x07, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, // "Enabled" + 0x01, // true + + 0x41, 0x82, 0x06, 0x46, 0x61, 0x63, 0x74, 0x6f, 0x72, // "Factor" + 0x03, 0xc0, 0x20, 0x80, // 8.25 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x0c, 0x52, 0x6f, 0x73, 0x65, 0x20, 0x4c, 0x61, 0x6c, 0x6f, 0x6e, 0x64, 0x65, // "Rose Lalonde" + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x02, 0x01, 0x9d, // 413 + }, testStructManyFields{ + Value: 413, + Name: "Rose Lalonde", + Enabled: true, + Factor: 8.25, + + hidden: nil, + inc: 0, + enabled: nil, + }, nil, 59) + + runDecTests(t, "with embedded", nil, + []byte{ + 0x01, 0x30, // len=48 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4e, 0x45, 0x50, 0x45, 0x54, 0x41, // "NEPETA" + + 0x41, 0x82, 0x11, 0x54, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x6f, 0x45, 0x6d, 0x62, 0x65, 0x64, // "TestStructToEmbed" + 0x01, 0x0a, // len=10 + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x04, // 4 + }, testStructWithEmbedded{ + TestStructToEmbed: TestStructToEmbed{ + Value: 4, + }, + Name: "NEPETA", + }, nil, 50) + + runDecTests(t, "with embedded overlap", nil, + []byte{ + 0x01, 0x3c, // len=60 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4e, 0x45, 0x50, 0x45, 0x54, 0x41, // "NEPETA" + + 0x41, 0x82, 0x11, 0x54, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x6f, 0x45, 0x6d, 0x62, 0x65, 0x64, // "TestStructToEmbed" + 0x01, 0x0a, // len=10 + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x01, 0x04, // 4 + + 0x41, 0x82, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, // "Value" + 0x03, 0xc0, 0x20, 0x80, // 8.25 + }, testStructWithEmbeddedOverlap{ + TestStructToEmbed: TestStructToEmbed{ + Value: 4, + }, + Value: 8.25, + Name: "NEPETA", + }, nil, 62) + + // // specialized test we cannot abstract easily + t.Run("missing values in encoded are left alone", func(t *testing.T) { + assert := assert.New(t) + + var ( + actual = testStructMultiMember{Value: 8, Name: "JOHN"} + input = []byte{ + 0x01, 0x10, // len=16 + + 0x41, 0x82, 0x04, 0x4e, 0x61, 0x6d, 0x65, // "Name" + 0x41, 0x82, 0x06, 0x4e, 0x45, 0x50, 0x45, 0x54, 0x41, // "NEPETA" + } + expect = testStructMultiMember{Value: 8, Name: "NEPETA"} + expectConsumed = 18 + ) + + consumed, err := Dec(input, &actual) + if !assert.NoError(err) { + return + } + + assert.Equal(expect, actual, "value mismatch") + assert.Equal(expectConsumed, consumed, "consumed bytes mismatch") + }) +} + +// expectConsumed used only in sub-tests where expect is the actual expected. +// +// if initVal is nil it will be set to an empty value +func runDecTests[E any](t *testing.T, name string, initVal *E, filledInput []byte, filledExpect E, emptyExpect *E, filledExpectConsumed int) { + // TODO: too complicated, just make the test cases instead of going off to a function + + // normal value test + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + var ( + actual E + input = filledInput + expect = filledExpect + expectConsumed = filledExpectConsumed + ) + + if initVal != nil { + actual = *initVal + } + + consumed, err := Dec(input, &actual) + if !assert.NoError(err) { + return + } + + assert.Equal(expect, actual, "value mismatch") + assert.Equal(expectConsumed, consumed, "consumed bytes mismatch") + }) + + // 0-len struct + t.Run(name+", no values encoded", func(t *testing.T) { + assert := assert.New(t) + + var ( + actual E + input = []byte{0x00} + expect E + expectConsumed = 1 + ) + + if initVal != nil { + actual = *initVal + } + + if emptyExpect != nil { + expect = *emptyExpect + } + + consumed, err := Dec(input, &actual) + if !assert.NoError(err) { + return + } + + assert.Equal(expect, actual, "value mismatch") + assert.Equal(expectConsumed, consumed, "consumed bytes mismatch") + }) + + // single pointer, filled + t.Run("*("+name+")", func(t *testing.T) { + assert := assert.New(t) + + var ( + actual *E + input = filledInput + expectVal = filledExpect + expect = &expectVal + expectConsumed = filledExpectConsumed + ) + + if initVal != nil { + actual = new(E) + *actual = (*initVal) + } + + consumed, err := Dec(input, &actual) + if !assert.NoError(err) { + return + } + + assert.Equal(expect, actual, "value mismatch") + assert.Equal(expectConsumed, consumed, "consumed bytes mismatch") + }) + + // single pointer, nil + t.Run("*("+name+"), nil", func(t *testing.T) { + assert := assert.New(t) + + var ( + actual = &filledExpect // initially set to enshore it's cleared + input = []byte{0xa0} + expect *E + expectConsumed = 1 + ) + + consumed, err := Dec(input, &actual) + if !assert.NoError(err) { + return + } + + assert.Equal(expect, actual, "value mismatch") + assert.Equal(expectConsumed, consumed, "consumed bytes mismatch") + }) + + // double pointer, filled + t.Run("**("+name+")", func(t *testing.T) { + assert := assert.New(t) + + var ( + actual **E + input = filledInput + expectVal = filledExpect + expectPtr = &expectVal + expect = &expectPtr + expectConsumed = filledExpectConsumed + ) + + if initVal != nil { + actual = new(*E) + *actual = new(E) + **actual = (*initVal) + } + + consumed, err := Dec(input, &actual) + if !assert.NoError(err) { + return + } + + assert.Equal(expect, actual, "value mismatch") + assert.Equal(expectConsumed, consumed, "consumed bytes mismatch") + }) + + // double pointer, nil at first level + t.Run("**("+name+"), nil at first level", func(t *testing.T) { + assert := assert.New(t) + + var ( + actualInitialPtr = &filledExpect + actual = &actualInitialPtr // initially set to enshore it's cleared + input = []byte{0xb0, 0x01, 0x01} + expectPtr *E + expect = &expectPtr + expectConsumed = 3 + ) + + consumed, err := Dec(input, &actual) + if !assert.NoError(err) { + return + } + + assert.Equal(expect, actual, "value mismatch") + assert.Equal(expectConsumed, consumed, "consumed bytes mismatch") + }) +} diff --git a/typeinfo.go b/typeinfo.go index 3bd7316..d5922eb 100644 --- a/typeinfo.go +++ b/typeinfo.go @@ -4,6 +4,7 @@ import ( "encoding" "fmt" "reflect" + "sort" ) var ( @@ -47,6 +48,7 @@ const ( mtComplex mtArray mtText + mtStruct ) func (mt mainType) String() string { @@ -75,11 +77,74 @@ func (mt mainType) String() string { return "mtArray" case mtText: return "mtText" + case mtStruct: + return "mtStruct" default: return fmt.Sprintf("mainType(%d)", mt) } } +// fieldInfo holds REZI-specific type info on fieldds of a struct +type fieldInfo struct { + Name string + Index int // position in fields by index + Type typeInfo + Anonymous bool // TODO: check if this actually needed on completion of #61 +} + +type fields struct { + ByName map[string]fieldInfo + ByOrder []fieldInfo +} + +// sortableFields can sort a slice of fieldInfo. select whether by Name or by +// Index with the alpha property. +type sortableFields struct { + fields []fieldInfo + alpha bool // if alpha is false, it's sorted by Index of the fields in fs. else by Name. +} + +// Len implements sort.Interface +func (sf *sortableFields) Len() int { + return len(sf.fields) +} + +// Less implements sort.Interface +func (sf *sortableFields) Less(i, j int) bool { + f1 := sf.fields[i] + f2 := sf.fields[j] + + if sf.alpha { + return f1.Name < f2.Name + } + return f1.Index < f2.Index +} + +// Swap implements sort.Interface +func (sf *sortableFields) Swap(i, j int) { + f1 := sf.fields[i] + f2 := sf.fields[j] + + sf.fields[i] = f2 + sf.fields[j] = f1 +} + +// does not sort in place; makes complete copy +func sortFieldsByName(fields []fieldInfo) []fieldInfo { + sorting := &sortableFields{fields: make([]fieldInfo, len(fields)), alpha: true} + copy(sorting.fields, fields) + sort.Sort(sorting) + return sorting.fields +} + +// does not sort in place; makes complete copy +func sortFieldsByIndex(fields []fieldInfo) []fieldInfo { + sorting := &sortableFields{fields: make([]fieldInfo, len(fields)), alpha: false} + copy(sorting.fields, fields) + sort.Sort(sorting) + return sorting.fields +} + // typeInfo holds REZI-specific type info on types that can be encoded and // decoded. type typeInfo struct { @@ -92,6 +157,7 @@ type typeInfo struct { ValType *typeInfo // valid for map, slice, and array Len int // only valid for array Dec bool // whether the info is for a decoded value. if false, it's for an encoded one. + Fields fields // valid for struct only } // MainReflectType returns the reflect.Type that represents the main type of the @@ -183,6 +249,19 @@ func (ti typeInfo) MainReflectType(indirected bool) reflect.Type { t = reflect.SliceOf(vrt) case mtString: t = refPrimitiveKindTypes[reflect.String] + case mtStruct: + sorted := sortFieldsByIndex(ti.Fields.ByOrder) + refFields := []reflect.StructField{} + for _, fi := range sorted { + structRefType := fi.Type.MainReflectType(true) + sf := reflect.StructField{ + Name: fi.Name, + Anonymous: fi.Anonymous, + Type: structRefType, + } + refFields = append(refFields, sf) + } + t = reflect.StructOf(refFields) } if t != nil && indirected { @@ -337,6 +416,26 @@ func encTypeInfo(t reflect.Type) (info typeInfo, err error) { } arrLen := t.Len() return typeInfo{Indir: indirCount, Main: mtArray, ValType: &arrValInfo, Len: arrLen}, nil + case reflect.Struct: + // could be okay, but all exported fields must be encodable. + // check while building lists of fields + fieldsData := fields{ByName: map[string]fieldInfo{}} + + for i := 0; i < t.NumField(); i++ { + sf := t.Field(i) + if !sf.IsExported() { + continue + } + fieldValInfo, err := encTypeInfo(sf.Type) + if err != nil { + return typeInfo{}, errorf("field .%s is not encodeable: %s", sf.Name, err) + } + fi := fieldInfo{Index: i, Name: sf.Name, Anonymous: sf.Anonymous, Type: fieldValInfo} + fieldsData.ByName[fi.Name] = fi + fieldsData.ByOrder = append(fieldsData.ByOrder, fi) + } + fieldsData.ByOrder = sortFieldsByName(fieldsData.ByOrder) + return typeInfo{Indir: indirCount, Main: mtStruct, Fields: fieldsData}, nil case reflect.Pointer: // try removing one level of indrection and checking THAT t = t.Elem() @@ -468,6 +567,27 @@ func decTypeInfo(t reflect.Type) (info typeInfo, err error) { } arrLen := t.Len() return typeInfo{Dec: true, Indir: indirCount, Underlying: under, Main: mtArray, ValType: &arrValInfo, Len: arrLen}, nil + case reflect.Struct: + // could be okay, but all exported fields must be encodable. + // check while building lists of fields + fieldsData := fields{ByName: map[string]fieldInfo{}} + + for i := 0; i < t.NumField(); i++ { + sf := t.Field(i) + if !sf.IsExported() { + continue + } + fieldValInfo, err := decTypeInfo(sf.Type) + if err != nil { + return typeInfo{}, errorf("field .%s is not decodeable: %s", sf.Name, err) + } + fi := fieldInfo{Index: i, Name: sf.Name, Anonymous: sf.Anonymous, Type: fieldValInfo} + fieldsData.ByName[fi.Name] = fi + fieldsData.ByOrder = append(fieldsData.ByOrder, fi) + } + fieldsData.ByOrder = sortFieldsByName(fieldsData.ByOrder) + // doesn't make sense to set Underlying for a struct; it will ALWAYS be the 'underlying' type. + return typeInfo{Dec: true, Indir: indirCount, Main: mtStruct, Fields: fieldsData}, nil case reflect.Pointer: // try removing one level of indrection and checking THAT t = t.Elem()