Decode/Unmarshal a msgpack array of arbitrary length into a struct (new directive:vartuple)

gen/decode.go

@@ -73,12 +73,14 @@ func (d *decodeGen) assignAndCheck(name string, typ string) {
 }
 
 func (d *decodeGen) structAsTuple(s *Struct) {
-	nfields := len(s.Fields)
-
 	sz := randIdent()
 	d.p.declare(sz, u32)
 	d.assignAndCheck(sz, arrayHeader)
-	d.p.arrayCheck(strconv.Itoa(nfields), sz)
+	if s.AsVarTuple {
+		d.p.printf("\nif %[1]s == 0 { return }", sz)
+	} else {
+		d.p.arrayCheck(strconv.Itoa(len(s.Fields)), sz)
+	}
 	for i := range s.Fields {
 		if !d.p.ok() {
 			return
@@ -98,6 +100,12 @@ func (d *decodeGen) structAsTuple(s *Struct) {
 		if anField {
 			d.p.printf("\n}") // close if statement
 		}
+		if s.AsVarTuple {
+			d.p.printf("\nif %[1]s--; %[1]s == 0 { return }", sz)
+		}
+	}
+	if s.AsVarTuple {
+		d.p.printf("\nfor ; %[1]s > 0; %[1]s-- {\nif err = dc.Skip(); err != nil {\nerr = msgp.WrapError(err)\nreturn\n}\n}", sz)
 	}
 }
 

gen/elem.go

@@ -459,8 +459,9 @@ func (s *Ptr) IfZeroExpr() string { return s.Varname() + " == nil" }
 
 type Struct struct {
 	common
-	Fields  []StructField // field list
-	AsTuple bool          // write as an array instead of a map
+	Fields     []StructField // field list
+	AsTuple    bool          // write as an array instead of a map
+	AsVarTuple bool          // write as an array of variable length instead of a map
 }
 
 func (s *Struct) TypeName() string {

gen/encode.go

@@ -97,9 +97,7 @@ func (e *encodeGen) tuple(s *Struct) {
 	data := msgp.AppendArrayHeader(nil, uint32(nfields))
 	e.p.printf("\n// array header, size %d", nfields)
 	e.Fuse(data)
-	if len(s.Fields) == 0 {
-		e.fuseHook()
-	}
+	e.fuseHook()
 	for i := range s.Fields {
 		if !e.p.ok() {
 			return

gen/marshal.go

@@ -104,9 +104,7 @@ func (m *marshalGen) tuple(s *Struct) {
 	data = msgp.AppendArrayHeader(data, uint32(len(s.Fields)))
 	m.p.printf("\n// array header, size %d", len(s.Fields))
 	m.Fuse(data)
-	if len(s.Fields) == 0 {
-		m.fuseHook()
-	}
+	m.fuseHook()
 	for i := range s.Fields {
 		if !m.p.ok() {
 			return

gen/unmarshal.go

@@ -77,7 +77,11 @@ func (u *unmarshalGen) tuple(s *Struct) {
 	sz := randIdent()
 	u.p.declare(sz, u32)
 	u.assignAndCheck(sz, arrayHeader)
-	u.p.arrayCheck(strconv.Itoa(len(s.Fields)), sz)
+	if s.AsVarTuple {
+		u.p.printf("\nif %[1]s == 0 {\no = bts\nreturn\n}", sz)
+	} else {
+		u.p.arrayCheck(strconv.Itoa(len(s.Fields)), sz)
+	}
 	for i := range s.Fields {
 		if !u.p.ok() {
 			return
@@ -101,6 +105,12 @@ func (u *unmarshalGen) tuple(s *Struct) {
 		if anField {
 			u.p.printf("\n}")
 		}
+		if s.AsVarTuple {
+			u.p.printf("\nif %[1]s--; %[1]s == 0 {\no = bts\nreturn\n}", sz)
+		}
+	}
+	if s.AsVarTuple {
+		u.p.printf("\nfor ; %[1]s > 0; %[1]s-- {\nbts, err = msgp.Skip(bts)\nif err != nil {\nerr = msgp.WrapError(err)\nreturn\n}\n}", sz)
 	}
 }
 

helper_test.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/tinylib/msgp/gen"
+)
+
+const showGeneratedFile = false
+
+func generate(t *testing.T, content string) (string, error) {
+	tempDir := t.TempDir()
+
+	mainFilename := filepath.Join(tempDir, "main.go")
+
+	fd, err := os.OpenFile(mainFilename, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o600)
+	if err != nil {
+		return "", err
+	}
+	defer fd.Close()
+
+	if _, err := fd.WriteString(content); err != nil {
+		return "", err
+	}
+
+	mode := gen.Encode | gen.Decode | gen.Size | gen.Marshal | gen.Unmarshal | gen.Test
+	if err := Run(mainFilename, mode, false); err != nil {
+		return "", err
+	}
+
+	if showGeneratedFile {
+		mainGenFilename := strings.TrimSuffix(mainFilename, ".go") + "_gen.go"
+		content, err := os.ReadFile(mainGenFilename)
+		if err != nil {
+			return "", err
+		}
+		t.Logf("generated %s content:\n%s", mainGenFilename, content)
+	}
+
+	return mainFilename, nil
+}
+
+func goExec(t *testing.T, mainFilename string, test bool) {
+	mainGenFilename := strings.TrimSuffix(mainFilename, ".go") + "_gen.go"
+
+	args := []string{"run", mainFilename, mainGenFilename}
+	if test {
+		mainGenTestFilename := strings.TrimSuffix(mainFilename, ".go") + "_gen_test.go"
+		args = []string{"test", mainFilename, mainGenFilename, mainGenTestFilename}
+	}
+
+	output, err := exec.Command("go", args...).CombinedOutput()
+	if err != nil {
+		t.Fatalf("go run failed: %v, output:\n%s", err, output)
+	}
+}

issue275_test.go

@@ -0,0 +1,127 @@
+package main
+
+import "testing"
+
+func TestIssue275Tuples(t *testing.T) {
+	mainFilename, err := generate(t, issue275Tuples)
+	if err != nil {
+		t.Fatalf("generate failed: %v", err)
+	}
+	goExec(t, mainFilename, false) // exec go run
+	goExec(t, mainFilename, true)  // exec go test
+}
+
+var issue275Tuples = `package main
+
+import (
+	"fmt"
+	"os"
+)
+
+//go:generate msgp
+
+//msgp:tuple Test1
+type Test1 struct {
+	Foo string
+}
+
+//msgp:tuple Test2
+type Test2 struct {
+	Foo string
+	Bar string
+}
+
+//msgp:tuple Test3
+type Test3 struct {
+	Foo string
+	Bar string
+	Baz string
+}
+
+//msgp:vartuple Test
+type Test struct {
+	Foo string
+	Bar string
+}
+
+func main() {
+	t1 := Test1{Foo: "Foo1"}
+	d1, err := t1.MarshalMsg(nil)
+	if err != nil {
+		fmt.Println("Test1 MarshalMsg failed:", err)
+		os.Exit(1)
+	}
+
+	t2 := Test2{Foo: "Foo2", Bar: "Bar2"}
+	d2, err := t2.MarshalMsg(nil)
+	if err != nil {
+		fmt.Println("Test2 MarshalMsg failed:", err)
+		os.Exit(1)
+	}
+
+	t3 := Test3{Foo: "Foo3", Bar: "Bar3", Baz: "Baz3"}
+	d3, err := t3.MarshalMsg(nil)
+	if err != nil {
+		fmt.Println("Test3 MarshalMsg failed:", err)
+		os.Exit(1)
+	}
+
+	var msg Test
+
+	msg = Test{}
+	if _, err := msg.UnmarshalMsg(d1); err != nil {
+		fmt.Println("Test UnmarshalMsg from Test1 failed:", err)
+		os.Exit(1)
+	}
+	if msg.Foo != "Foo1" {
+		fmt.Println("Test UnmarshalMsg from Test1 bad msg:", msg)
+		os.Exit(1)
+	}
+
+	msg = Test{}
+	if _, err := msg.UnmarshalMsg(d2); err != nil {
+		fmt.Println("Test UnmarshalMsg from Test2 failed:", err)
+		os.Exit(1)
+	}
+	if msg.Foo != "Foo2" || msg.Bar != "Bar2" {
+		fmt.Println("Test UnmarshalMsg from Test2 bad msg:", msg)
+		os.Exit(1)
+	}
+
+	msg = Test{}
+	if _, err := msg.UnmarshalMsg(d3); err != nil {
+		fmt.Println("Test UnmarshalMsg from Test3 failed:", err)
+		os.Exit(1)
+	}
+	if msg.Foo != "Foo3" || msg.Bar != "Bar3" {
+		fmt.Println("Test UnmarshalMsg from Test3 bad msg:", msg)
+		os.Exit(1)
+	}
+
+	var msg1 Test1
+	if _, err := msg1.UnmarshalMsg(d2); err != nil {
+		if err.Error() != "msgp: wanted array of size 1; got 2" {
+			fmt.Println("Test1 UnmarshalMsg from Test2 failed:", err)
+			os.Exit(1)
+		}
+	}
+
+	var msg2 Test2
+	if _, err := msg2.UnmarshalMsg(d2); err != nil {
+		fmt.Println("Test2 UnmarshalMsg from Test2 failed:", err)
+		os.Exit(1)
+	}
+	if msg2.Foo != "Foo2" || msg2.Bar != "Bar2" {
+		fmt.Println("Test3 UnmarshalMsg from Test2 bad msg:", msg)
+		os.Exit(1)
+	}
+
+	var msg3 Test3
+	if _, err := msg3.UnmarshalMsg(d2); err != nil {
+		if err.Error() != "msgp: wanted array of size 3; got 2" {
+			fmt.Println("Test3 UnmarshalMsg from Test2 failed:", err)
+			os.Exit(1)
+		}
+	}
+}
+`

issue395_test.go

@@ -0,0 +1,50 @@
+package main
+
+import "testing"
+
+func TestIssue395TupleAllownil(t *testing.T) {
+	mainFilename, err := generate(t, issue395TupleAllownil)
+	if err != nil {
+		t.Fatalf("generate failed: %v", err)
+	}
+
+	goExec(t, mainFilename, false) // exec go run
+	goExec(t, mainFilename, true)  // exec go test
+}
+
+var issue395TupleAllownil = `package main
+
+import (
+	"fmt"
+	"os"
+)
+
+//go:generate msgp
+
+//msgp:tuple User
+type User struct {
+	ID       []byte ` + "`msgpack:\",allownil\"`" + `
+	Name     string
+	Email    string
+	IsActive bool
+}
+
+func main() {
+	u := User{ID: nil, Name: "user"}
+	data, err := u.MarshalMsg(nil)
+	if err != nil {
+		fmt.Println("User MarshalMsg failed:", err)
+		os.Exit(1)
+	}
+
+	var user User
+	if _, err := user.UnmarshalMsg(data); err != nil {
+		fmt.Println("User UnmarshalMsg failed:", err)
+		os.Exit(1)
+	}
+	if user.ID != nil || user.Name != "user" {
+		fmt.Println("User UnmarshalMsg bad user:", user)
+		os.Exit(1)
+	}
+}
+`

parse/directives.go

@@ -26,6 +26,7 @@ var directives = map[string]directive{
 	"replace":       replace,
 	"ignore":        ignore,
 	"tuple":         astuple,
+	"vartuple":      asvartuple,
 	"compactfloats": compactfloats,
 	"clearomitted":  clearomitted,
 	"newtime":       newtime,
@@ -176,6 +177,26 @@ func astuple(text []string, f *FileSet) error {
 	return nil
 }
 
+//msgp:vartuple {TypeA} {TypeB}...
+func asvartuple(text []string, f *FileSet) error {
+	if len(text) < 2 {
+		return nil
+	}
+	for _, item := range text[1:] {
+		name := strings.TrimSpace(item)
+		if el, ok := f.Identities[name]; ok {
+			if st, ok := el.(*gen.Struct); ok {
+				st.AsTuple = true
+				st.AsVarTuple = true
+				infof(name)
+			} else {
+				warnf("%s: only structs can be tuples\n", name)
+			}
+		}
+	}
+	return nil
+}
+
 //msgp:tag {tagname}
 func tag(text []string, f *FileSet) error {
 	if len(text) != 2 {