Decoder hooks (#3)

* decoder hooks

* add comment in decoder hook test

decode.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"math"
 	"reflect"
+	"strconv"
 	"time"
 )
 
@@ -35,6 +36,9 @@ type parser struct {
 	anchors  map[string]*Node
 	doneInit bool
 	textless bool
+
+	hook     DecoderHook
+	hookPath []string
 }
 
 func newParser(b []byte) *parser {
@@ -58,12 +62,17 @@ func newParserFromReader(r io.Reader) *parser {
 	return &p
 }
 
+func (p *parser) withHook(h DecoderHook) {
+	p.hook = h
+}
+
 func (p *parser) init() {
 	if p.doneInit {
 		return
 	}
 	p.anchors = make(map[string]*Node)
 	p.expect(yaml_STREAM_START_EVENT)
+	p.hookPath = make([]string, 0)
 	p.doneInit = true
 }
 
@@ -139,19 +148,23 @@ func (p *parser) anchor(n *Node, anchor []byte) {
 	}
 }
 
-func (p *parser) parse() *Node {
+func (p *parser) parse(triggerHook bool) *Node {
 	p.init()
+	var n *Node
 	switch p.peek() {
 	case yaml_SCALAR_EVENT:
-		return p.scalar()
+		n = p.scalar()
 	case yaml_ALIAS_EVENT:
-		return p.alias()
+		n = p.alias()
 	case yaml_MAPPING_START_EVENT:
-		return p.mapping()
+		triggerHook = false // maps are not leaf nodes, skip hook
+		n = p.mapping()
 	case yaml_SEQUENCE_START_EVENT:
-		return p.sequence()
+		triggerHook = false // sequences are not leaf nodes, skip hook
+		n = p.sequence()
 	case yaml_DOCUMENT_START_EVENT:
-		return p.document()
+		triggerHook = false // documents are not leaf nodes, skip hook
+		n =  p.document()
 	case yaml_STREAM_END_EVENT:
 		// Happens when attempting to decode an empty buffer.
 		return nil
@@ -160,6 +173,11 @@ func (p *parser) parse() *Node {
 	default:
 		panic("internal error: attempted to parse unknown event (please report): " + p.event.typ.String())
 	}
+
+	if triggerHook && p.hook != nil {
+		p.hook(p.hookPath, n)
+	}
+	return n
 }
 
 func (p *parser) node(kind Kind, defaultTag, tag, value string) *Node {
@@ -188,8 +206,8 @@ func (p *parser) node(kind Kind, defaultTag, tag, value string) *Node {
 	return n
 }
 
-func (p *parser) parseChild(parent *Node) *Node {
-	child := p.parse()
+func (p *parser) parseChild(parent *Node, hook bool) *Node {
+	child := p.parse(hook)
 	parent.Content = append(parent.Content, child)
 	return child
 }
@@ -198,7 +216,7 @@ func (p *parser) document() *Node {
 	n := p.node(DocumentNode, "", "", "")
 	p.doc = n
 	p.expect(yaml_DOCUMENT_START_EVENT)
-	p.parseChild(n)
+	p.parseChild(n, false)
 	if p.peek() == yaml_DOCUMENT_END_EVENT {
 		n.FootComment = string(p.event.foot_comment)
 	}
@@ -254,7 +272,9 @@ func (p *parser) sequence() *Node {
 	p.anchor(n, p.event.anchor)
 	p.expect(yaml_SEQUENCE_START_EVENT)
 	for p.peek() != yaml_SEQUENCE_END_EVENT {
-		p.parseChild(n)
+		p.hookPath = append(p.hookPath, strconv.Itoa(len(n.Content)))
+		p.parseChild(n, true)
+		p.hookPath = p.hookPath[:len(p.hookPath)-1]
 	}
 	n.LineComment = string(p.event.line_comment)
 	n.FootComment = string(p.event.foot_comment)
@@ -272,15 +292,16 @@ func (p *parser) mapping() *Node {
 	p.anchor(n, p.event.anchor)
 	p.expect(yaml_MAPPING_START_EVENT)
 	for p.peek() != yaml_MAPPING_END_EVENT {
-		k := p.parseChild(n)
+		k := p.parseChild(n, false)
+		p.hookPath = append(p.hookPath, k.Value)
 		if block && k.FootComment != "" {
 			// Must be a foot comment for the prior value when being dedented.
 			if len(n.Content) > 2 {
 				n.Content[len(n.Content)-3].FootComment = k.FootComment
 				k.FootComment = ""
 			}
 		}
-		v := p.parseChild(n)
+		v := p.parseChild(n, true)
 		if k.FootComment == "" && v.FootComment != "" {
 			k.FootComment = v.FootComment
 			v.FootComment = ""
@@ -291,6 +312,7 @@ func (p *parser) mapping() *Node {
 			}
 			p.expect(yaml_TAIL_COMMENT_EVENT)
 		}
+		p.hookPath = p.hookPath[:len(p.hookPath)-1]
 	}
 	n.LineComment = string(p.event.line_comment)
 	n.FootComment = string(p.event.foot_comment)

decode_test.go

@@ -25,8 +25,8 @@ import (
 	"strings"
 	"time"
 
-	. "gopkg.in/check.v1"
 	"github.com/conduitio/yaml/v3"
+	. "gopkg.in/check.v1"
 )
 
 var unmarshalIntTest = 123
@@ -1715,6 +1715,61 @@ func (s *S) TestUnmarshalKnownFields(c *C) {
 	}
 }
 
+func (s *S) TestDecoderHook(c *C) {
+	y := `
+a:
+  b: foo
+  c:
+    d:
+      e: bar
+f:
+  g:
+    h:
+      i:
+        - 1
+        - 2
+j: {"k": true, "l": false}
+
+x:
+  - z: yes
+    y: no
+  - z: up
+    w: down
+`
+	got := make(map[string]*yaml.Node)
+
+	dec := yaml.NewDecoder(bytes.NewBufferString(y))
+	dec.WithHook(func(path []string, node *yaml.Node) {
+		joinedPath := strings.Join(path, ".")
+		if _, ok := got[joinedPath]; ok {
+			c.Fatalf("got path twice: %v", joinedPath)
+		}
+		got[joinedPath] = node
+	})
+	err := dec.Decode(map[string]interface{}{})
+	c.Assert(err, IsNil)
+
+	want := map[string]string{
+		"a.b": "foo",
+		"a.c.d.e": "bar",
+		"f.g.h.i.0": "1",
+		"f.g.h.i.1": "2",
+		"j.k": "true",
+		"j.l": "false",
+		"x.0.z": "yes",
+		"x.0.y": "no",
+		"x.1.z": "up",
+		"x.1.w": "down",
+	}
+	c.Assert(len(got), Equals, len(want))
+	for k,v := range want {
+		comment := Commentf("key: %s", k)
+		node, ok := got[k]
+		c.Assert(ok, Equals, true, comment)
+		c.Assert(node.Value, Equals, v, comment)
+	}
+}
+
 type textUnmarshaler struct {
 	S string
 }

yaml.go

@@ -87,7 +87,22 @@ type Marshaler interface {
 // supported tag options.
 //
 func Unmarshal(in []byte, out interface{}) (err error) {
-	return unmarshal(in, out, false)
+	defer handleErr(&err)
+	d := newDecoder()
+	p := newParser(in)
+	defer p.destroy()
+	node := p.parse(false)
+	if node != nil {
+		v := reflect.ValueOf(out)
+		if v.Kind() == reflect.Ptr && !v.IsNil() {
+			v = v.Elem()
+		}
+		d.unmarshal(node, v)
+	}
+	if len(d.terrors) > 0 {
+		return &TypeError{d.terrors}
+	}
+	return nil
 }
 
 // A Decoder reads and decodes YAML values from an input stream.
@@ -112,6 +127,15 @@ func (dec *Decoder) KnownFields(enable bool) {
 	dec.knownFields = enable
 }
 
+// DecoderHook is called for every leaf node and receives the node itself and
+// the path leading up to the node.
+type DecoderHook func(path []string, node *Node)
+
+// WithHook adds a DecoderHook that gets called for every leaf node.
+func (dec *Decoder) WithHook(h DecoderHook) {
+	dec.parser.withHook(h)
+}
+
 // Decode reads the next YAML-encoded value from its input
 // and stores it in the value pointed to by v.
 //
@@ -121,7 +145,7 @@ func (dec *Decoder) Decode(v interface{}) (err error) {
 	d := newDecoder()
 	d.knownFields = dec.knownFields
 	defer handleErr(&err)
-	node := dec.parser.parse()
+	node := dec.parser.parse(false)
 	if node == nil {
 		return io.EOF
 	}
@@ -154,25 +178,6 @@ func (n *Node) Decode(v interface{}) (err error) {
 	return nil
 }
 
-func unmarshal(in []byte, out interface{}, strict bool) (err error) {
-	defer handleErr(&err)
-	d := newDecoder()
-	p := newParser(in)
-	defer p.destroy()
-	node := p.parse()
-	if node != nil {
-		v := reflect.ValueOf(out)
-		if v.Kind() == reflect.Ptr && !v.IsNil() {
-			v = v.Elem()
-		}
-		d.unmarshal(node, v)
-	}
-	if len(d.terrors) > 0 {
-		return &TypeError{d.terrors}
-	}
-	return nil
-}
-
 // Marshal serializes the value provided into a YAML document. The structure
 // of the generated document will reflect the structure of the value itself.
 // Maps and pointers (to struct, string, int, etc) are accepted as the in value.
@@ -266,7 +271,7 @@ func (n *Node) Encode(v interface{}) (err error) {
 	p := newParser(e.out)
 	p.textless = true
 	defer p.destroy()
-	doc := p.parse()
+	doc := p.parse(false)
 	*n = *doc.Content[0]
 	return nil
 }
@@ -517,9 +522,9 @@ const (
 // control over the content being decoded or encoded.
 //
 // It's worth noting that although Node offers access into details such as
-// line numbers, colums, and comments, the content when re-encoded will not
+// line numbers, columns, and comments, the content when re-encoded will not
 // have its original textual representation preserved. An effort is made to
-// render the data plesantly, and to preserve comments near the data they
+// render the data pleasantly, and to preserve comments near the data they
 // describe, though.
 //
 // Values that make use of the Node type interact with the yaml package in the
@@ -545,7 +550,7 @@ type Node struct {
 	// scalar nodes may be obtained via the ShortTag and LongTag methods.
 	Kind  Kind
 
-	// Style allows customizing the apperance of the node in the tree.
+	// Style allows customizing the appearance of the node in the tree.
 	Style Style
 
 	// Tag holds the YAML tag defining the data type for the value.
@@ -557,7 +562,7 @@ type Node struct {
 	// the implicit tag diverges from the provided one.
 	Tag string
 
-	// Value holds the unescaped and unquoted represenation of the value.
+	// Value holds the unescaped and unquoted representation of the value.
 	Value string
 
 	// Anchor holds the anchor name for this node, which allows aliases to point to it.