Skip to content

Commit

Permalink
Adds a hook to specify unmarshal options.
Browse files Browse the repository at this point in the history
This makes it possible to configure the parser to error out if
an unknown tagged field has been found in the byte stream
that is being unmarshaled to a go object.

NOTE: This change requires go 1.10.  It seems that we should all
upgrade, see: https://pocketgophers.com/when-should-you-upgrade-go/

For example like this (also see ExampleUnknown in yaml_test.go):

```
func ExampleUnknown() {
	type WithTaggedField struct {
		Field string `json:"field"`
	}
	y := []byte(`unknown: "hello"`)
	v := WithTaggedField{}
	fmt.Printf("%v\n", Unmarshal(y, &v, DisallowUnknownFields))
	// Ouptut:
	// unmarshaling JSON: while decoding JSON: json: unknown field "unknown"
}
```

If you want to manipulate the decoder used in the unmarshal function,
you can define a custom option to your liking and apply it in the
Unmarshal call.

The way I found out about this is kind of comical.  Some nonprintable
characters made their way into my YAML file.  You wouldn't see them in the
editor, but they'd be there.  Now when the YAML file is parsed, the
nonprintable chars together with the key characters were made into a
weird UTF-8 key (e.g. like "\u00c2some_key"), which would then
become a key unknown to the tags of my struct, and would silently get
dropped in the parsing steps.  So as result, you would get a perfectly
normal parse, except the data that you wanted to be in the struct are not
there.

This meant that silently dropping unknown keys is not always a good
idea, so I went to fix it, while retaining the same interface.  JSONOpt
type was fairly easy to add in the Unmarshal because it can be silently
ignored by the old code.
  • Loading branch information
filmil committed Apr 9, 2018
1 parent 0ca9ea5 commit 8aa0426
Show file tree
Hide file tree
Showing 5 changed files with 88 additions and 7 deletions.
5 changes: 3 additions & 2 deletions .travis.yml
@@ -1,7 +1,8 @@
language: go
go:
- 1.3
- 1.4
- "1.3"
- "1.4"
- "1.10"
script:
- go test
- go build
26 changes: 23 additions & 3 deletions yaml.go
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"encoding/json"
"fmt"
"io"
"reflect"
"strconv"

Expand All @@ -26,22 +27,41 @@ func Marshal(o interface{}) ([]byte, error) {
return y, nil
}

// Converts YAML to JSON then uses JSON to unmarshal into an object.
func Unmarshal(y []byte, o interface{}) error {
// JSONOpt is a decoding option for decoding from JSON format.
type JSONOpt func(*json.Decoder) *json.Decoder

// Unmarshal converts YAML to JSON then uses JSON to unmarshal into an object,
// optionally configuring the behavior of the JSON unmarshal.
func Unmarshal(y []byte, o interface{}, opts ...JSONOpt) error {
vo := reflect.ValueOf(o)
j, err := yamlToJSON(y, &vo)
if err != nil {
return fmt.Errorf("error converting YAML to JSON: %v", err)
}

err = json.Unmarshal(j, o)
err = jsonUnmarshal(bytes.NewReader(j), o, opts...)
if err != nil {
return fmt.Errorf("error unmarshaling JSON: %v", err)
}

return nil
}

// jsonUnmarshal unmarshals the JSON byte stream from the given reader into the
// object, optionally applying decoder options prior to decoding. We are not
// using json.Unmarshal directly as we want the chance to pass in non-default
// options.
func jsonUnmarshal(r io.Reader, o interface{}, opts ...JSONOpt) error {
d := json.NewDecoder(r)
for _, opt := range opts {
d = opt(d)
}
if err := d.Decode(&o); err != nil {
return fmt.Errorf("while decoding JSON: %v", err)
}
return nil
}

// Convert JSON to YAML.
func JSONToYAML(j []byte) ([]byte, error) {
// Convert the JSON to an object.
Expand Down
14 changes: 14 additions & 0 deletions yaml_go110.go
@@ -0,0 +1,14 @@
// This file contains changes that are only compatible with go 1.10 and onwards.

// +build go1.10

package yaml

import "encoding/json"

// DisallowUnknownFields configures the JSON decoder to error out if unknown
// fields come along, instead of dropping them by default.
func DisallowUnknownFields(d *json.Decoder) *json.Decoder {
d.DisallowUnknownFields()
return d
}
46 changes: 46 additions & 0 deletions yaml_go110_test.go
@@ -0,0 +1,46 @@
// +build go1.10

package yaml

import (
"fmt"
"testing"
)

func TestUnmarshalWithTags(t *testing.T) {
type WithTaggedField struct {
Field string `json:"field"`
}

t.Run("Known tagged field", func(t *testing.T) {
y := []byte(`field: "hello"`)
v := WithTaggedField{}
if err := Unmarshal(y, &v, DisallowUnknownFields); err != nil {
t.Errorf("unexpected error: %v", err)
}
if v.Field != "hello" {
t.Errorf("v.Field=%v, want 'hello'", v.Field)
}

})
t.Run("With unknown tagged field", func(t *testing.T) {
y := []byte(`unknown: "hello"`)
v := WithTaggedField{}
err := Unmarshal(y, &v, DisallowUnknownFields)
if err == nil {
t.Errorf("want error because of unknown field, got <nil>: v=%#v", v)
}
})

}

func ExampleUnknown() {
type WithTaggedField struct {
Field string `json:"field"`
}
y := []byte(`unknown: "hello"`)
v := WithTaggedField{}
fmt.Printf("%v\n", Unmarshal(y, &v, DisallowUnknownFields))
// Ouptut:
// unmarshaling JSON: while decoding JSON: json: unknown field "unknown"
}
4 changes: 2 additions & 2 deletions yaml_test.go
Expand Up @@ -106,8 +106,8 @@ b:
unmarshal(t, y, &s5, &e5)
}

func unmarshal(t *testing.T, y []byte, s, e interface{}) {
err := Unmarshal(y, s)
func unmarshal(t *testing.T, y []byte, s, e interface{}, opts ...JSONOpt) {
err := Unmarshal(y, s, opts...)
if err != nil {
t.Errorf("error unmarshaling YAML: %v", err)
}
Expand Down

0 comments on commit 8aa0426

Please sign in to comment.