cue/scanner: fix JSON conformance

Comma and colon may start on a new line.

See https://tools.ietf.org/html/rfc7159#section-2.

Fixes #721

Change-Id: Iae6fd9e15faf239a64b74cc3f404a704b796b570
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/8564
Reviewed-by: Marcel van Lohuizen <mpvl@golang.org>

cue/parser/parser_test.go

@@ -602,6 +602,15 @@ bar: 2
 			// Comment3
 		)`,
 		out: "funcArg1: foo(<[1// Comment1] {}>, <[d0// Comment2] [d1// Comment3] {}>)",
+	}, {
+		desc: "front-style commas",
+		in: `
+			frontStyle: { "key": "value"
+				, "key2": "value2"
+				, "foo" : bar
+			}
+			`,
+		out: "frontStyle: {\"key\": \"value\", \"key2\": \"value2\", \"foo\": bar}",
 	}}
 	for _, tc := range testCases {
 		t.Run(tc.desc, func(t *testing.T) {

cue/parser/testdata/commas.src

@@ -30,3 +30,6 @@ bar: [
 	3
 ]
 
+frontStyle: { "key": "value"
+	, "key2": "value2"
+}

cue/scanner/scanner.go

@@ -831,7 +831,15 @@ scanAgain:
 			// set in the first place and exited early
 			// from s.skipWhitespace()
 			s.insertEOL = false // newline consumed
-			return s.file.Pos(offset, token.Elided), token.COMMA, "\n"
+			p := s.file.Pos(offset, token.Elided)
+			s.skipWhitespace(1)
+			// Don't elide comma before a ',' or ':' to ensure JSON
+			// conformance. Note that cue fmt should immediately undo those.
+			if s.ch == ',' || s.ch == ':' {
+				return s.Scan()
+			}
+			return p, token.COMMA, "\n"
+
 		case '#':
 			for quote.numHash++; s.ch == '#'; quote.numHash++ {
 				s.next()

cue/scanner/scanner_test.go

@@ -461,7 +461,9 @@ func TestRelative(t *testing.T) {
 	b :    5
 	// line one
 	// line two
-	c: "dfs"
+	c
+	: "dfs"
+	, d: "foo"
 	`
 	want := []string{
 		`newline IDENT    package`,
@@ -480,8 +482,12 @@ func TestRelative(t *testing.T) {
 		"newline COMMENT  // line one",
 		"newline COMMENT  // line two",
 		`newline IDENT    c`,
-		`nospace :        `,
+		`newline :        `,
 		`blank   STRING   "dfs"`,
+		"newline ,        ,",
+		"blank   IDENT    d",
+		"nospace :        ",
+		`blank   STRING   "foo"`,
 		"elided  ,        \n",
 	}
 	var S Scanner

cue/testdata/compile/json.txtar

@@ -0,0 +1,50 @@
+Issue #721
+
+-- in.cue --
+// allow front-style commas
+a
+: { "key": "value"
+, "key2"
+: "value2"
+}
+
+, b: [
+    0
+,   1
+,       2
+        ,3,
+    4
+
+    , 5
+]
+-- out/compile --
+--- in.cue
+{
+  a: {
+    key: "value"
+    key2: "value2"
+  }
+  b: [
+    0,
+    1,
+    2,
+    3,
+    4,
+    5,
+  ]
+}
+-- out/eval --
+(struct){
+  a: (struct){
+    key: (string){ "value" }
+    key2: (string){ "value2" }
+  }
+  b: (#list){
+    0: (int){ 0 }
+    1: (int){ 1 }
+    2: (int){ 2 }
+    3: (int){ 3 }
+    4: (int){ 4 }
+    5: (int){ 5 }
+  }
+}