encoding/yaml: fix bug where an empty document is not treated as null

Change-Id: Ibb763b5e44d6c80d569db8df863d79631d0d2935
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/6561
Reviewed-by: CUE cueckoo <cueckoo@gmail.com>
Reviewed-by: Marcel van Lohuizen <mpvl@golang.org>

encoding/yaml/yaml.go

@@ -37,11 +37,14 @@ func Extract(filename string, src interface{}) (*ast.File, error) {
 	}
 	for {
 		expr, err := d.Decode()
-		if err == io.EOF {
-			break
-		}
 		if err != nil {
-			return nil, err
+			if err != io.EOF {
+				return nil, err
+			}
+			if expr != nil {
+				a = append(a, expr)
+			}
+			break
 		}
 		a = append(a, expr)
 	}

encoding/yaml/yaml_test.go

@@ -27,9 +27,19 @@ func TestYAML(t *testing.T) {
 	testCases := []struct {
 		name     string
 		yaml     string
+		yamlOut  string
 		want     string
 		isStream bool
 	}{{
+		name:    "empty",
+		yaml:    "",
+		yamlOut: "null",
+		want:    "null",
+	}, {
+		name:     "empty stream",
+		want:     "null",
+		isStream: true,
+	}, {
 		name: "string literal",
 		yaml: `foo`,
 		want: `"foo"`,
@@ -54,7 +64,37 @@ c: baz
 }]`,
 		isStream: true,
 	}, {
-		name:     "emtpy",
+		name: "stream with null",
+		yaml: `
+---
+a: foo
+---
+---
+b: bar
+c: baz
+---
+`,
+		// Not sure if a leading document separator should be gobbled, but the
+		// YAML parser seems to think so. This could have something to do with
+		// the fact that the document separator is really an "end of directives"
+		// marker, while ... means "end of document". YAML is hard!
+		yamlOut: `a: foo
+---
+null
+---
+b: bar
+c: baz
+---
+null
+`,
+		// TODO(bug): seems like bug in yaml parser. Try moving to yaml.v3,
+		// or validate that this is indeed a correct interpretation.
+		want: `[{
+	a: "foo"
+}, null, {
+	b: "bar"
+	c: "baz"
+}, null]`,
 		isStream: true,
 	}}
 	r := &cue.Runtime{}
@@ -66,7 +106,7 @@ c: baz
 			}
 			b, _ := format.Node(f)
 			if got := strings.TrimSpace(string(b)); got != tc.want {
-				t.Errorf("got %q; want %q", got, tc.want)
+				t.Errorf("Extract:\ngot  %q\nwant %q", got, tc.want)
 			}
 
 			inst, err := Decode(r, tc.name, tc.yaml)
@@ -79,7 +119,12 @@ c: baz
 			}
 			b, _ = format.Node(n)
 			if got := strings.TrimSpace(string(b)); got != tc.want {
-				t.Errorf("got %q; want %q", got, tc.want)
+				t.Errorf("Decode:\ngot  %q\nwant %q", got, tc.want)
+			}
+
+			yamlOut := tc.yaml
+			if tc.yamlOut != "" {
+				yamlOut = tc.yamlOut
 			}
 
 			inst, _ = r.Compile(tc.name, tc.want)
@@ -88,17 +133,17 @@ c: baz
 				if err != nil {
 					t.Error(err)
 				}
-				if got := strings.TrimSpace(string(b)); got != tc.yaml {
-					t.Errorf("got %q; want %q", got, tc.yaml)
+				if got := strings.TrimSpace(string(b)); got != yamlOut {
+					t.Errorf("Encode:\ngot  %q\nwant %q", got, yamlOut)
 				}
 			} else {
 				iter, _ := inst.Value().List()
 				b, err := EncodeStream(iter)
 				if err != nil {
 					t.Error(err)
 				}
-				if got := string(b); got != tc.yaml {
-					t.Errorf("got %q; want %q", got, tc.yaml)
+				if got := string(b); got != yamlOut {
+					t.Errorf("EncodeStream:\ngot  %q\nwant %q", got, yamlOut)
 				}
 			}
 		})

internal/third_party/yaml/yaml.go

@@ -86,8 +86,9 @@ func Unmarshal(filename string, in []byte) (expr ast.Expr, err error) {
 
 // A Decorder reads and decodes YAML values from an input stream.
 type Decoder struct {
-	strict bool
-	parser *parser
+	strict    bool
+	firstDone bool
+	parser    *parser
 }
 
 // NewDecoder returns a new decoder that reads from r.
@@ -113,8 +114,12 @@ func (dec *Decoder) Decode() (expr ast.Expr, err error) {
 	defer handleErr(&err)
 	node := dec.parser.parse()
 	if node == nil {
-		return nil, io.EOF
+		if !dec.firstDone {
+			expr = ast.NewNull()
+		}
+		return expr, io.EOF
 	}
+	dec.firstDone = true
 	expr = d.unmarshal(node)
 	if len(d.terrors) > 0 {
 		return nil, &TypeError{d.terrors}