Permalink
Browse files

Allow usage of message types with customtype override (#239)

  • Loading branch information...
jvshahid authored and awalterschulze committed Jan 4, 2017
1 parent 48d61f4 commit f9114dace7bd920b32f943b3c73fafbcbab2bf31
Showing with 32,291 additions and 9,679 deletions.
  1. +1 −0 CONTRIBUTORS
  2. +68 −0 custom_types.md
  3. +1 −11 extensions.md
  4. +20 −0 jsonpb/jsonpb.go
  5. +1 −1 plugin/gostring/gostring.go
  6. +17 −17 plugin/populate/populate.go
  7. +1 −1 plugin/stringer/stringer.go
  8. +9 −3 plugin/unmarshal/unmarshal.go
  9. +1 −1 protoc-gen-gogo/generator/generator.go
  10. +77 −0 test/combos/both/t.go
  11. +2,697 −798 test/combos/both/thetest.pb.go
  12. +23 −0 test/combos/both/thetest.proto
  13. +1,754 −176 test/combos/both/thetestpb_test.go
  14. +77 −0 test/combos/marshaler/t.go
  15. +2,050 −645 test/combos/marshaler/thetest.pb.go
  16. +23 −0 test/combos/marshaler/thetest.proto
  17. +1,754 −176 test/combos/marshaler/thetestpb_test.go
  18. +73 −0 test/combos/unmarshaler/t.go
  19. +2,513 −798 test/combos/unmarshaler/thetest.pb.go
  20. +23 −0 test/combos/unmarshaler/thetest.proto
  21. +1,586 −176 test/combos/unmarshaler/thetestpb_test.go
  22. +77 −0 test/combos/unsafeboth/t.go
  23. +2,697 −798 test/combos/unsafeboth/thetest.pb.go
  24. +23 −0 test/combos/unsafeboth/thetest.proto
  25. +1,754 −176 test/combos/unsafeboth/thetestpb_test.go
  26. +77 −0 test/combos/unsafemarshaler/t.go
  27. +2,071 −665 test/combos/unsafemarshaler/thetest.pb.go
  28. +23 −0 test/combos/unsafemarshaler/thetest.proto
  29. +1,754 −176 test/combos/unsafemarshaler/thetestpb_test.go
  30. +73 −0 test/combos/unsafeunmarshaler/t.go
  31. +2,514 −798 test/combos/unsafeunmarshaler/thetest.pb.go
  32. +23 −0 test/combos/unsafeunmarshaler/thetest.proto
  33. +1,586 −176 test/combos/unsafeunmarshaler/thetestpb_test.go
  34. +73 −0 test/t.go
  35. +479 −474 test/theproto3/combos/both/theproto3.pb.go
  36. +474 −469 test/theproto3/combos/marshaler/theproto3.pb.go
  37. +480 −475 test/theproto3/combos/neither/theproto3.pb.go
  38. +475 −470 test/theproto3/combos/unmarshaler/theproto3.pb.go
  39. +481 −476 test/theproto3/combos/unsafeboth/theproto3.pb.go
  40. +481 −476 test/theproto3/combos/unsafemarshaler/theproto3.pb.go
  41. +481 −476 test/theproto3/combos/unsafeunmarshaler/theproto3.pb.go
  42. +1,817 −595 test/thetest.pb.go
  43. +23 −0 test/thetest.proto
  44. +1,586 −176 test/thetestpb_test.go
View
@@ -6,6 +6,7 @@ Dwayne Schultz <dschultz@pivotal.io>
Georg Apitz <gapitz@pivotal.io>
Gustav Paul <gustav.paul@gmail.com>
Johan Brandhorst <johan.brandhorst@gmail.com>
John Shahid <jvshahid@gmail.com>
John Tuley <john@tuley.org>
Laurent <laurent@adyoulike.com>
Patrick Lee <patrick@dropbox.com>
View
@@ -0,0 +1,68 @@
# Custom types
Custom types is a gogo protobuf extensions that allows for using a custom
struct type to decorate the underlying structure of the protocol message.
# How to use
## Defining the protobuf message
```proto
message CustomType {
optional ProtoType Field = 1 [(gogoproto.customtype) = "T"];
}
message ProtoType {
optional string Field = 1;
}
```
or alternatively you can declare the field type in the protocol message to be
`bytes`:
```proto
message BytesCustomType {
optional bytes Field = 1 [(gogoproto.customtype) = "T"];
}
```
The downside of using `bytes` is that it makes it harder to generate protobuf
code in other languages. In either case, it is the user responsibility to
ensure that the custom type marshals and unmarshals to the expected wire
format. That is, in the first example, gogo protobuf will not attempt to ensure
that the wire format of `ProtoType` and `T` are wire compatible.
## Custom type method signatures
The custom type must define the following methods with the given
signatures. Assuming the custom type is called `T`:
```go
func (t T) Marshal() ([]byte, error) {}
func (t *T) MarshalTo(data []byte) (n int, err error) {}
func (t *T) Unmarshal(data []byte) error {}
func (t T) MarshalJSON() ([]byte, error) {}
func (t *T) UnmarshalJSON(data []byte) error {}
// only required if the compare option is set
func (t T) Compare(other T) int {}
// only required if the equal option is set
func (t T) Equal(other T) bool {}
// only required if populate option is set
func NewPopulatedT(r randyThetest) *T {}
```
Check [t.go](test/t.go) for a full example
# Warnings and issues
`Warning about customtype: It is your responsibility to test all cases of your marshaling, unmarshaling and size methods implemented for your custom type.`
Issues with customtype include:
* <a href="https://github.com/gogo/protobuf/issues/199">A Bytes method is not allowed.<a/>
* <a href="https://github.com/gogo/protobuf/issues/132">Defining a customtype as a fake proto message is broken.</a>
* <a href="https://github.com/gogo/protobuf/issues/147">proto.Clone is broken.</a>
* <a href="https://github.com/gogo/protobuf/issues/125">Using a proto message as a customtype is not allowed.</a>
* <a href="https://github.com/gogo/protobuf/issues/200">cusomtype of type map can not UnmarshalText</a>
* <a href="https://github.com/gogo/protobuf/issues/201">customtype of type struct cannot jsonpb unmarshal</a>
View
@@ -37,7 +37,7 @@ You might also find that basic structs that started their life as part of an API
<tr><td><b>Name</b></td><td><b>Option</b></td><td><b>Type</b></td><td><b>Description</b></td><td><b>Default</b></td></tr>
<tr><td><a href="http://godoc.org/github.com/gogo/protobuf/gogoproto">nullable</a></td><td> Field </td><td> bool </td><td> if false, a field is generated without a pointer (see warning below). </td><td> true </td></tr>
<tr><td><a href="http://godoc.org/github.com/gogo/protobuf/gogoproto">embed</a></td><td> Field </td><td> bool </td><td> if true, the field is generated as an embedded field. </td><td> false </td></tr>
<tr><td><a href="http://godoc.org/github.com/gogo/protobuf/gogoproto">customtype</a> </td><td> Field </td><td> string </td><td> It works with the Marshal and Unmarshal methods, to allow you to have your own types in your struct, but marshal to bytes. For example, custom.Uuid or custom.Fixed128 </td><td> goprotobuf type </td></tr>
<tr><td><a href="http://godoc.org/github.com/gogo/protobuf/gogoproto">customtype</a> </td><td> Field </td><td> string </td><td> It works with the Marshal and Unmarshal methods, to allow you to have your own types in your struct, but marshal to bytes. For example, custom.Uuid or custom.Fixed128. For more information please refer to the [CustomTypes](custom_types.md) document </td><td> goprotobuf type </td></tr>
<tr><td><a href="http://godoc.org/github.com/gogo/protobuf/gogoproto"> customname</a> (beta) </td><td> Field </td><td> string </td><td> Changes the generated fieldname. This is especially useful when generated methods conflict with fieldnames. </td><td> goprotobuf field name </td></tr>
<tr><td><a href="http://godoc.org/github.com/gogo/protobuf/gogoproto"> casttype</a> (beta) </td><td> Field </td><td> string </td><td> Changes the generated field type. It assumes that this type is castable to the original goprotobuf field type. It currently does not support maps, structs or enums. </td><td> goprotobuf field type </td></tr>
<tr><td><a href="http://godoc.org/github.com/gogo/protobuf/gogoproto"> castkey </a> (beta) </td><td> Field </td><td> string </td><td> Changes the generated fieldtype for a map key. All generated code assumes that this type is castable to the protocol buffer field type. Only supported on maps. </td><td> goprotobuf field type </td></tr>
@@ -50,16 +50,6 @@ You might also find that basic structs that started their life as part of an API
`Warning about nullable: according to the Protocol Buffer specification, you should be able to tell whether a field is set or unset. With the option nullable=false this feature is lost, since your non-nullable fields will always be set.`
`Warning about customtype: It is your responsibility to test all cases of your marshaling, unmarshaling and size methods implemented for your custom type.`
Issues with customtype include:
* <a href="https://github.com/gogo/protobuf/issues/199">A Bytes method is not allowed.<a/>
* <a href="https://github.com/gogo/protobuf/issues/132">Defining a customtype as a fake proto message is broken.</a>
* <a href="https://github.com/gogo/protobuf/issues/147">proto.Clone is broken.</a>
* <a href="https://github.com/gogo/protobuf/issues/125">Using a proto message as a customtype is not allowed.</a>
* <a href="https://github.com/gogo/protobuf/issues/200">cusomtype of type map can not UnmarshalText</a>
* <a href="https://github.com/gogo/protobuf/issues/201">customtype of type struct cannot jsonpb unmarshal</a>
# Goprotobuf Compatibility
Gogoprotobuf is compatible with Goprotobuf, because it is compatible with protocol buffers (see the section on tests below).
View
@@ -494,6 +494,18 @@ func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v refle
out.write(`null`)
return out.err
}
if m, ok := v.Interface().(interface {
MarshalJSON() ([]byte, error)
}); ok {
data, err := m.MarshalJSON()
if err != nil {
return err
}
out.write(string(data))
return nil
}
pm, ok := iface.(proto.Message)
if !ok {
if prop.CustomType == "" {
@@ -739,6 +751,14 @@ func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMe
// Handle nested messages.
if targetType.Kind() == reflect.Struct {
if target.CanAddr() {
if m, ok := target.Addr().Interface().(interface {
UnmarshalJSON([]byte) error
}); ok {
return json.Unmarshal(inputValue, m)
}
}
var jsonFields map[string]json.RawMessage
if err := json.Unmarshal(inputValue, &jsonFields); err != nil {
return err
@@ -224,7 +224,7 @@ func (p *gostring) Generate(file *generator.FileDescriptor) {
p.P(`s = append(s, "`, fieldname, `: " + `, mapName, `+ ",\n")`)
p.Out()
p.P(`}`)
} else if field.IsMessage() || p.IsGroup(field) {
} else if (field.IsMessage() && !gogoproto.IsCustomType(field)) || p.IsGroup(field) {
if nullable || repeated {
p.P(`if this.`, fieldname, ` != nil {`)
p.In()
@@ -305,6 +305,23 @@ func (p *plugin) GenerateField(file *generator.FileDescriptor, message *generato
}
p.Out()
p.P(`}`)
} else if gogoproto.IsCustomType(field) {
funcCall := p.getCustomFuncCall(goTypName)
if field.IsRepeated() {
p.P(p.varGen.Next(), ` := r.Intn(10)`)
p.P(`this.`, fieldname, ` = make(`, goTyp, `, `, p.varGen.Current(), `)`)
p.P(`for i := 0; i < `, p.varGen.Current(), `; i++ {`)
p.In()
p.P(p.varGen.Next(), `:= `, funcCall)
p.P(`this.`, fieldname, `[i] = *`, p.varGen.Current())
p.Out()
p.P(`}`)
} else if gogoproto.IsNullable(field) {
p.P(`this.`, fieldname, ` = `, funcCall)
} else {
p.P(p.varGen.Next(), `:= `, funcCall)
p.P(`this.`, fieldname, ` = *`, p.varGen.Current())
}
} else if field.IsMessage() || p.IsGroup(field) {
funcCall := p.getFuncCall(goTypName)
if field.IsRepeated() {
@@ -345,23 +362,6 @@ func (p *plugin) GenerateField(file *generator.FileDescriptor, message *generato
p.P(p.varGen.Next(), ` := `, val)
p.P(`this.`, fieldname, ` = &`, p.varGen.Current())
}
} else if gogoproto.IsCustomType(field) {
funcCall := p.getCustomFuncCall(goTypName)
if field.IsRepeated() {
p.P(p.varGen.Next(), ` := r.Intn(10)`)
p.P(`this.`, fieldname, ` = make(`, goTyp, `, `, p.varGen.Current(), `)`)
p.P(`for i := 0; i < `, p.varGen.Current(), `; i++ {`)
p.In()
p.P(p.varGen.Next(), `:= `, funcCall)
p.P(`this.`, fieldname, `[i] = *`, p.varGen.Current())
p.Out()
p.P(`}`)
} else if gogoproto.IsNullable(field) {
p.P(`this.`, fieldname, ` = `, funcCall)
} else {
p.P(p.varGen.Next(), `:= `, funcCall)
p.P(`this.`, fieldname, ` = *`, p.varGen.Current())
}
} else if field.IsBytes() {
if field.IsRepeated() {
p.P(p.varGen.Next(), ` := r.Intn(10)`)
@@ -203,7 +203,7 @@ func (p *stringer) Generate(file *generator.FileDescriptor) {
} else if p.IsMap(field) {
mapName := `mapStringFor` + fieldname
p.P("`", fieldname, ":`", ` + `, mapName, " + `,", "`,")
} else if field.IsMessage() || p.IsGroup(field) {
} else if (field.IsMessage() && !gogoproto.IsCustomType(field)) || p.IsGroup(field) {
desc := p.ObjectNamed(field.GetTypeName())
msgname := p.TypeName(desc)
msgnames := strings.Split(msgname, ".")
@@ -806,10 +806,13 @@ func (p *unmarshal) field(file *generator.FileDescriptor, msg *generator.Descrip
} else {
p.P(`m.`, fieldname, ` = append(m.`, fieldname, `, time.Duration(0))`)
}
} else if nullable {
} else if nullable && !gogoproto.IsCustomType(field) {
p.P(`m.`, fieldname, ` = append(m.`, fieldname, `, &`, msgname, `{})`)
} else {
p.P(`m.`, fieldname, ` = append(m.`, fieldname, `, `, msgname, `{})`)
goType, _ := p.GoType(nil, field)
// remove the slice from the type, i.e. []*T -> *T
goType = goType[2:]
p.P(`m.`, fieldname, ` = append(m.`, fieldname, `, `, goType, `{})`)
}
varName := `m.` + fieldname + `[len(m.` + fieldname + `)-1]`
buf := `dAtA[iNdEx:postIndex]`
@@ -840,7 +843,9 @@ func (p *unmarshal) field(file *generator.FileDescriptor, msg *generator.Descrip
} else if gogoproto.IsStdDuration(field) {
p.P(`m.`, fieldname, ` = new(time.Duration)`)
} else {
p.P(`m.`, fieldname, ` = &`, msgname, `{}`)
goType, _ := p.GoType(nil, field)
// remove the star from the type
p.P(`m.`, fieldname, ` = &`, goType[1:], `{}`)
}
p.Out()
p.P(`}`)
@@ -869,6 +874,7 @@ func (p *unmarshal) field(file *generator.FileDescriptor, msg *generator.Descrip
p.P(`}`)
}
p.P(`iNdEx = postIndex`)
case descriptor.FieldDescriptorProto_TYPE_BYTES:
p.P(`var byteLen int`)
p.decodeVarint("byteLen", "int")
@@ -1769,7 +1769,7 @@ func (g *Generator) goTag(message *Descriptor, field *descriptor.FieldDescriptor
func needsStar(field *descriptor.FieldDescriptorProto, proto3 bool, allowOneOf bool) bool {
if isRepeated(field) &&
(*field.Type != descriptor.FieldDescriptorProto_TYPE_MESSAGE) &&
(*field.Type != descriptor.FieldDescriptorProto_TYPE_MESSAGE || gogoproto.IsCustomType(field)) &&
(*field.Type != descriptor.FieldDescriptorProto_TYPE_GROUP) {
return false
}
View
@@ -0,0 +1,77 @@
package test
import (
"encoding/json"
"strings"
"github.com/gogo/protobuf/proto"
)
type T struct {
Data string
}
func (gt *T) protoType() *ProtoType {
return &ProtoType{
Field2: &gt.Data,
}
}
func (gt T) Equal(other T) bool {
return gt.protoType().Equal(other.protoType())
}
func (gt *T) Size() int {
proto := &ProtoType{
Field2: &gt.Data,
}
return proto.Size()
}
func NewPopulatedT(r randyThetest) *T {
data := NewPopulatedProtoType(r, false).Field2
gt := &T{}
if data != nil {
gt.Data = *data
}
return gt
}
func (r T) Marshal() ([]byte, error) {
return proto.Marshal(r.protoType())
}
func (r *T) MarshalTo(data []byte) (n int, err error) {
return r.protoType().MarshalTo(data)
}
func (r *T) Unmarshal(data []byte) error {
pr := &ProtoType{}
err := proto.Unmarshal(data, pr)
if err != nil {
return err
}
if pr.Field2 != nil {
r.Data = *pr.Field2
}
return nil
}
func (gt T) MarshalJSON() ([]byte, error) {
return json.Marshal(gt.Data)
}
func (gt *T) UnmarshalJSON(data []byte) error {
var s string
err := json.Unmarshal(data, &s)
if err != nil {
return err
}
*gt = T{Data: s}
return nil
}
func (gt T) Compare(other T) int {
return strings.Compare(gt.Data, other.Data)
}
Oops, something went wrong.

0 comments on commit f9114da

Please sign in to comment.