encoding/yaml: quote strings more aggressively

Rather than always quoting strings, quote any string that matches any of
the regexpes in the tag repository.
The algorithm is rather aggressive with
matching and matches a superset of
these tags.

This implementation is an alternative to the suggestion in Issue #1368.
If this is not sufficient, we can still address that.

Fixes #1076
Issue #1368

Signed-off-by: Marcel van Lohuizen <mpvl@golang.org>

Change-Id: I03e4b820a0f90b594feffce8a0bccbb3ffda35e1
Signed-off-by: Marcel van Lohuizen <mpvl@golang.org>
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/527375
Unity-Result: CUEcueckoo <cueckoo@cuelang.org>
TryBot-Result: CUEcueckoo <cueckoo@cuelang.org>
Reviewed-by: Paul Jolly <paul@myitcv.io>

Author: mpvl

.github/workflows/new_version_triggers.yml

@@ -1,7 +1,7 @@
 # Generated by internal/ci/ci_tool.cue; do not edit
 
 name: New release triggers
-on:
+"on":
   push:
     tags:
     - v*

.github/workflows/release.yml

@@ -1,7 +1,7 @@
 # Generated by internal/ci/ci_tool.cue; do not edit
 
 name: Release
-on:
+"on":
   push:
     tags:
     - v*

.github/workflows/repository_dispatch.yml

@@ -1,7 +1,7 @@
 # Generated by internal/ci/ci_tool.cue; do not edit
 
 name: Repository Dispatch
-on:
+"on":
 - repository_dispatch
 jobs:
   runtrybot:

.github/workflows/test.yml

@@ -1,7 +1,7 @@
 # Generated by internal/ci/ci_tool.cue; do not edit
 
 name: Test
-on:
+"on":
   push:
     branches:
     - '**'

.github/workflows/tip_triggers.yml

@@ -1,7 +1,7 @@
 # Generated by internal/ci/ci_tool.cue; do not edit
 
 name: Push to tip triggers
-on:
+"on":
   push:
     branches:
     - master

cmd/cue/cmd/testdata/script/cmd_github.txt

@@ -51,7 +51,7 @@ jobs:
 # Generated by internal/ci/ci_tool.cue; do not edit
 
 name: New release triggers
-on:
+"on":
   push:
     tags:
     - v*
@@ -74,7 +74,7 @@ jobs:
 # Generated by internal/ci/ci_tool.cue; do not edit
 
 name: Release
-on:
+"on":
   push:
     tags:
     - v*
@@ -115,7 +115,7 @@ jobs:
 # Generated by internal/ci/ci_tool.cue; do not edit
 
 name: Repository Dispatch
-on:
+"on":
 - repository_dispatch
 jobs:
   runtrybot:
@@ -148,7 +148,7 @@ jobs:
 # Generated by internal/ci/ci_tool.cue; do not edit
 
 name: Test
-on:
+"on":
   push:
     branches:
     - '**'
@@ -290,7 +290,7 @@ jobs:
 # Generated by internal/ci/ci_tool.cue; do not edit
 
 name: Push to tip triggers
-on:
+"on":
   push:
     branches:
     - master

encoding/yaml/yaml_test.go

@@ -20,6 +20,7 @@ import (
 
 	"cuelang.org/go/cue"
 	"cuelang.org/go/cue/ast"
+	"cuelang.org/go/cue/cuecontext"
 	"cuelang.org/go/cue/format"
 )
 
@@ -149,3 +150,98 @@ null
 		})
 	}
 }
+
+func TestYAMLValues(t *testing.T) {
+	testCases := []struct {
+		cue  string
+		yaml string
+	}{
+		// strings
+		{`"""
+	single
+	"""`, "single"}, // TODO: CUE simplifies this.
+
+		{`"""
+	aaaa
+	bbbb
+	"""`, `|-
+  aaaa
+  bbbb`},
+
+		// keep as is
+		{`"non"`, `non`},
+
+		// Non-strings in v1.2. These are single-quoted by the go-yaml.v3 package.
+		{`"#cloudmon"`, `'#cloudmon'`},
+
+		// Strings that mimic numeric values are double quoted by the go-yaml.v3
+		// package.
+		{`".inf"`, `".inf"`},
+		{`".Inf"`, `".Inf"`},
+		{`".INF"`, `".INF"`},
+		{`".NaN"`, `".NaN"`},
+		{`"+.Inf"`, `"+.Inf"`},
+		{`"-.Inf"`, `"-.Inf"`},
+		{`"2002"`, `"2002"`},
+		{`"685_230.15"`, `"685_230.15"`},
+
+		// Legacy values.format.
+		{`"no"`, `"no"`},
+		{`"on"`, `"on"`},
+		{`".Nan"`, `".Nan"`},
+
+		// binary
+		{`'no'`, `!!binary bm8=`},
+
+		// floats
+		{`.2`, "0.2"},
+		{`2.`, "2."},
+		{`".inf"`, `".inf"`},
+		{`685_230.15`, `685230.15`},
+
+		// Date and time-like
+		{`"2001-12-15T02:59:43.1Z"`, `"2001-12-15T02:59:43.1Z"`},
+		{`"2001-12-14t21:59:43.10-05:00"`, `"2001-12-14t21:59:43.10-05:00"`},
+		{`"2001-12-14 21:59:43.10 -5"`, `"2001-12-14 21:59:43.10 -5"`},
+		{`"2001-12-15 2:59:43.10"`, `"2001-12-15 2:59:43.10"`},
+		{`"2002-12-14"`, `"2002-12-14"`},
+		{`"12-12-12"`, `"12-12-12"`},
+
+		// legacy base60 floats
+		{`"2222:22"`, `"2222:22"`},
+
+		// hostport
+		{`"hostname:22"`, `hostname:22`},
+
+		// maps
+		{
+			cue: `
+			true:   1
+			True:   2
+			".Nan": 3
+			".Inf": 4
+			y:      5
+			`,
+			yaml: `"true": 1
+"True": 2
+".Nan": 3
+".Inf": 4
+"y": 5`,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.cue, func(t *testing.T) {
+			c := cuecontext.New()
+			v := c.CompileString(tc.cue)
+
+			b, err := Encode(v)
+			if err != nil {
+				t.Error(err)
+			}
+			if got := strings.TrimSpace(string(b)); got != tc.yaml {
+				t.Errorf("Encode:\ngot  %q\nwant %q", got, tc.yaml)
+			}
+
+		})
+	}
+}

internal/encoding/yaml/encode.go

@@ -16,7 +16,10 @@ package yaml
 
 import (
 	"bytes"
+	"encoding/base64"
+	"fmt"
 	"math/big"
+	"regexp"
 	"strings"
 
 	"gopkg.in/yaml.v3"
@@ -101,6 +104,8 @@ func encode(n ast.Node) (y *yaml.Node, err error) {
 func encodeScalar(b *ast.BasicLit) (n *yaml.Node, err error) {
 	n = &yaml.Node{Kind: yaml.ScalarNode}
 
+	// TODO: use cue.Value and support attributes for setting YAML tags.
+
 	switch b.Kind {
 	case token.INT:
 		var x big.Int
@@ -118,18 +123,85 @@ func encodeScalar(b *ast.BasicLit) (n *yaml.Node, err error) {
 		n.Value = b.Value
 
 	case token.STRING:
-		str, err := literal.Unquote(b.Value)
+		info, nStart, _, err := literal.ParseQuotes(b.Value, b.Value)
 		if err != nil {
 			return nil, err
 		}
+		str, err := info.Unquote(b.Value[nStart:])
+		if err != nil {
+			panic(fmt.Sprintf("invalid string: %v", err))
+		}
 		n.SetString(str)
 
+		switch {
+		case !info.IsDouble():
+			n.Tag = "!!binary"
+			n.Value = base64.StdEncoding.EncodeToString([]byte(str))
+
+		case info.IsMulti():
+			// Preserve multi-line format.
+			n.Style = yaml.LiteralStyle
+
+		default:
+			if shouldQuote(str) {
+				n.Style = yaml.DoubleQuotedStyle
+			}
+		}
+
 	default:
 		return nil, errors.Newf(b.Pos(), "unknown literal type %v", b.Kind)
 	}
 	return n, nil
 }
 
+// shouldQuote indicates that a string may be a YAML 1.1. legacy value and that
+// the string should be quoted.
+func shouldQuote(str string) bool {
+	return legacyStrings[str] || useQuote.MatchString(str)
+}
+
+// This regular expression conservatively matches any date, time string,
+// or base60 float.
+var useQuote = regexp.MustCompile(`^[\-+0-9:\. \t]+([-:]|[tT])[\-+0-9:\. \t]+[zZ]?$`)
+
+// legacyStrings contains a map of fixed strings with special meaning for any
+// type in the YAML Tag registry (https://yaml.org/type/index.html) as used
+// in YAML 1.1.
+//
+// These strings are always quoted upon export to allow for backward
+// compatibility with YAML 1.1 parsers.
+var legacyStrings = map[string]bool{
+	"y":     true,
+	"Y":     true,
+	"yes":   true,
+	"Yes":   true,
+	"YES":   true,
+	"n":     true,
+	"N":     true,
+	"t":     true,
+	"T":     true,
+	"f":     true,
+	"F":     true,
+	"no":    true,
+	"No":    true,
+	"NO":    true,
+	"true":  true,
+	"True":  true,
+	"TRUE":  true,
+	"false": true,
+	"False": true,
+	"FALSE": true,
+	"on":    true,
+	"On":    true,
+	"ON":    true,
+	"off":   true,
+	"Off":   true,
+	"OFF":   true,
+
+	// Non-standard.
+	".Nan": true,
+}
+
 func setNum(n *yaml.Node, s string, x interface{}) error {
 	if yaml.Unmarshal([]byte(s), x) == nil {
 		n.Value = s
@@ -204,6 +276,9 @@ func encodeDecls(decls []ast.Decl) (n *yaml.Node, err error) {
 			label := &yaml.Node{}
 			addDocs(x.Label, label, label)
 			label.SetString(name)
+			if shouldQuote(name) {
+				label.Style = yaml.DoubleQuotedStyle
+			}
 
 			value, err := encode(x.Value)
 			if err != nil {

internal/encoding/yaml/encode_test.go

@@ -61,7 +61,7 @@ a:
     c: 3
 b:
   x: 0
-  y: 1
+  "y": 1
   z: 2
 		`,
 	}, {
@@ -89,7 +89,7 @@ hex: 0x11
 dec: .3
 dat: !!binary gA==
 nil: null
-yes: true
+"yes": true
 non: false
 `,
 	}, {