cue: implement "front-style" list comprehensions

This now also allows any of the non-JSON keywords
to be used as references. Previously, these were already
supported as field names.

Issue #339
Issue #165

Change-Id: I721d054c8220ba3536f680fe2e3e502a62f99b6b
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/5683
Reviewed-by: Marcel van Lohuizen <mpvl@golang.org>

cmd/cue/cmd/fix_test.go

@@ -53,6 +53,14 @@ a: {
 	X1="in": 3
 	ref:     X1 & x
 }
+`,
+	}, {
+		in: `
+		y: [1, 2, 3, 4]
+		a: [ x for x in y ]
+		`,
+		out: `y: [1, 2, 3, 4]
+a: [ for x in y { x } ]
 `,
 		// 	}, {
 		// 		name: "slice",

cmd/cue/cmd/testdata/script/legacy.txt

@@ -1,14 +1,16 @@
-! cue eval foo.cue
+cue eval foo.cue
 cmp stderr expect-stderr
 
-! cue export foo.cue
+cue export foo.cue
 cmp stderr expect-stderr
 
-! cue fmt foo.cue
-cmp stderr expect-stderr
+cue fmt foo.cue
+cmp foo.cue bar.cue
 
 -- expect-stderr --
-missing ',' in struct literal:
-    ./foo.cue:1:8
 -- foo.cue --
-"x": 3 for x in a
+a: [1]
+b: [ x for x in a ]
+-- bar.cue --
+a: [1]
+b: [ for x in a { x } ]

cue/ast.go

@@ -301,10 +301,39 @@ func (v *astVisitor) walk(astNode ast.Node) (ret value) {
 			parent:   v,
 		}
 
+		if len(n.Elts) == 1 {
+			if c, ok := n.Elts[0].(*ast.Comprehension); ok {
+				yielder := &yield{baseValue: newExpr(c.Value)}
+				lc := &listComprehension{
+					newExpr(c),
+					wrapClauses(v, yielder, c.Clauses),
+				}
+				// we don't support key for lists (yet?)
+
+				// TODO(hack): unwrap struct lit if embedding of one element.
+				// We do this as we do not yet support embedding of scalar
+				// values in general. This prohibits:
+				// - having fields alongside embedded values
+				// - having more than one embedded value.
+				// The latter would not be too hard to circumvent.
+				expr := c.Value
+				if s, ok := expr.(*ast.StructLit); ok && len(s.Elts) == 1 {
+					if e, ok := s.Elts[0].(*ast.EmbedDecl); ok {
+						expr = e.Expr
+					}
+				}
+				yielder.value = v.walk(expr)
+				return lc
+			}
+		}
+
 		elts, ellipsis := internal.ListEllipsis(n)
 
 		arcs := []arc{}
 		for i, e := range elts {
+			if _, ok := e.(*ast.Comprehension); ok {
+				return v.errf(e, "comprehensions must be a single element within list (for now)")
+			}
 			elem := v1.walk(e)
 			if elem == nil {
 				// TODO: it would be consistent to allow aliasing in lists

cue/ast/ast.go

@@ -347,10 +347,11 @@ func (a *Alias) End() token.Pos  { return a.Expr.End() }
 // A Comprehension node represents a comprehension declaration.
 type Comprehension struct {
 	Clauses []Clause // There must be at least one clause.
-	Value   Expr     // Must be a struct
+	Value   Expr     // Must be a struct TODO: change to Struct
 
 	comments
 	decl
+	expr // TODO: only allow Comprehension in "Embedding" productions.
 }
 
 func (x *Comprehension) Pos() token.Pos  { return getPos(x) }
@@ -551,6 +552,8 @@ func NewStruct(fields ...interface{}) *StructLit {
 // A ListLit node represents a literal list.
 type ListLit struct {
 	Lbrack token.Pos // position of "["
+
+	// TODO: change to embedding or similar.
 	Elts   []Expr    // list of composite elements; or nil
 	Rbrack token.Pos // position of "]"
 

cue/ast_test.go

@@ -313,7 +313,7 @@ alias "Y" redeclared in same scope:
 		out: `<0>{a: <1>{ <2>for k, v in <0>.b yield <3>{""+<2>.v+"": <2>.v}}, b: <4>{a: "aa", b: "bb", c: "cc"}}`,
 	}, {
 		in: `
-			a: [ v for _, v in b ]
+			a: [ for _, v in b { v } ]
 			b: { a: 1, b: 2, c: 3 }
 			`,
 		out: `<0>{a: [ <1>for _, v in <0>.b yield <1>.v ], b: <2>{a: 1, b: 2, c: 3}}`,

cue/export_test.go

@@ -251,11 +251,11 @@ func TestExport(t *testing.T) {
 			}`),
 	}, {
 		raw: true,
-		in:  `{ a: [1, 2], b: [ v for k, v in a ] }`,
+		in:  `{ a: [1, 2], b: [ for k, v in a { v }] }`,
 		out: unindent(`
 			{
 				a: [1, 2]
-				b: [ v for k, v in a ]
+				b: [ for k, v in a { v } ]
 			}`),
 	}, {
 		raw: true,

cue/format/node.go

@@ -184,7 +184,36 @@ func (f *formatter) walkClauseList(list []ast.Clause, ws whiteSpace) {
 	f.after(nil)
 }
 
-func (f *formatter) walkExprList(list []ast.Expr, depth int) {
+func (f *formatter) walkListElems(list []ast.Expr) {
+	f.before(nil)
+	for _, x := range list {
+		f.before(x)
+		switch n := x.(type) {
+		case *ast.Comprehension:
+			f.walkClauseList(n.Clauses, blank)
+			f.print(blank, nooverride)
+			f.expr(n.Value)
+
+		case *ast.Ellipsis:
+			f.ellipsis(n)
+
+		case *ast.Alias:
+			f.expr(n.Ident)
+			f.print(n.Equal, token.BIND)
+			f.expr(n.Expr)
+
+			// TODO: ast.CommentGroup: allows comment groups in ListLits.
+
+		case ast.Expr:
+			f.exprRaw(n, token.LowestPrec, 1)
+		}
+		f.print(comma, blank)
+		f.after(x)
+	}
+	f.after(nil)
+}
+
+func (f *formatter) walkArgsList(list []ast.Expr, depth int) {
 	f.before(nil)
 	for _, x := range list {
 		f.before(x)
@@ -235,7 +264,6 @@ func (f *formatter) inlineField(n *ast.Field) *ast.Field {
 }
 
 func (f *formatter) decl(decl ast.Decl) {
-
 	if decl == nil {
 		return
 	}
@@ -307,14 +335,6 @@ func (f *formatter) decl(decl ast.Decl) {
 			f.print(formfeed)
 		}
 
-	case *ast.Comprehension:
-		if !n.Pos().HasRelPos() || n.Pos().RelPos() >= token.Newline {
-			f.print(formfeed)
-		}
-		f.walkClauseList(n.Clauses, blank)
-		f.print(blank, nooverride)
-		f.expr(n.Value)
-
 	case *ast.BadDecl:
 		f.print(n.From, "*bad decl*", declcomma)
 
@@ -350,6 +370,29 @@ func (f *formatter) decl(decl ast.Decl) {
 		f.expr(n.Expr)
 		f.print(newline, noblank)
 
+	case *ast.Attribute:
+		f.print(n.At, n)
+
+	case *ast.CommentGroup:
+		f.print(newsection)
+		f.printComment(n)
+		f.print(newsection)
+
+	case ast.Expr:
+		f.embedding(n)
+	}
+}
+
+func (f *formatter) embedding(decl ast.Expr) {
+	switch n := decl.(type) {
+	case *ast.Comprehension:
+		if !n.Pos().HasRelPos() || n.Pos().RelPos() >= token.Newline {
+			f.print(formfeed)
+		}
+		f.walkClauseList(n.Clauses, blank)
+		f.print(blank, nooverride)
+		f.expr(n.Value)
+
 	case *ast.Ellipsis:
 		f.ellipsis(n)
 
@@ -362,13 +405,10 @@ func (f *formatter) decl(decl ast.Decl) {
 		f.expr(n.Expr)
 		f.print(declcomma) // implied
 
-	case *ast.Attribute:
-		f.print(n.At, n)
+		// TODO: ast.CommentGroup: allows comment groups in ListLits.
 
-	case *ast.CommentGroup:
-		f.print(newsection)
-		f.printComment(n)
-		f.print(newsection)
+	case ast.Expr:
+		f.exprRaw(n, token.LowestPrec, 1)
 	}
 }
 
@@ -570,7 +610,7 @@ func (f *formatter) exprRaw(expr ast.Expr, prec1, depth int) {
 		}
 		wasIndented := f.possibleSelectorExpr(x.Fun, token.HighestPrec, depth)
 		f.print(x.Lparen, token.LPAREN)
-		f.walkExprList(x.Args, depth)
+		f.walkArgsList(x.Args, depth)
 		f.print(trailcomma, noblank, x.Rparen, token.RPAREN)
 		if wasIndented {
 			f.print(unindent)
@@ -612,7 +652,7 @@ func (f *formatter) exprRaw(expr ast.Expr, prec1, depth int) {
 
 	case *ast.ListLit:
 		f.print(x.Lbrack, token.LBRACK, indent)
-		f.walkExprList(x.Elts, 1)
+		f.walkListElems(x.Elts)
 		f.print(trailcomma, noblank)
 		f.visitComments(f.current.pos)
 		f.matchUnindent()
@@ -623,9 +663,16 @@ func (f *formatter) exprRaw(expr ast.Expr, prec1, depth int) {
 
 	case *ast.ListComprehension:
 		f.print(x.Lbrack, token.LBRACK, blank, indent)
-		f.expr(x.Expr)
 		f.print(blank)
 		f.walkClauseList(x.Clauses, blank)
+		f.print(blank, nooverride)
+		if _, ok := x.Expr.(*ast.StructLit); ok {
+			f.expr(x.Expr)
+		} else {
+			f.print(token.LBRACE, blank)
+			f.expr(x.Expr)
+			f.print(blank, token.RBRACE)
+		}
 		f.print(unindent, f.wsOverride(blank), x.Rbrack, token.RBRACK)
 
 	default:

cue/format/testdata/expressions.golden

@@ -14,7 +14,7 @@ package expressions
 	alias = 3.14
 	"g\("en")"?: 4
 
-	alias2 = foo
+	alias2 = foo // with comment
 	aaalias = foo
 	b: bar
 
@@ -140,13 +140,16 @@ package expressions
 	e: [...int]
 	e: [...int]
 	e: [...int | float]
-	e: [ x for x in someObject if x > 9 ]
-	e: [ x
+	e: [ for x in someObject if x > 9 {
+		x
+	}]
+	e: [ for x in someObject if x > 9 { x } ]
+	e: [
 		for x in someObject
-		if x > 9 ]
-	e: [ x
+		if x > 9 { x } ]
+	e: [
 		for x in someObject
-		if x > 9
+		if x > 9 { x }
 	]
 	for k, v in someObject {
 		"\(k)": v

cue/format/testdata/expressions.input

@@ -14,7 +14,7 @@ package expressions
     alias = 3.14
     "g\("en")"?: 4
 
-    alias2 = foo
+    alias2 = foo // with comment
     aaalias = foo
     b: bar
 
@@ -138,6 +138,9 @@ package expressions
     e: [...int]
     e: [...int,]
     e: [...int | float]
+    e: [ for x in someObject if x > 9 {
+        x
+    }]
     e: [ x for x in someObject if x > 9 ]
     e: [ x
     for x in someObject
@@ -159,11 +162,11 @@ package expressions
             "\(k)":v
         }
     }
-
+    
     e: { for k, v in someObject if k > "a" {"\(k)":v} }
     e: { for k, v in someObject if k > "a" {
         "\(k)":v }}
-    
+
     e: {
     for k, v in someObject
     if k > "a" {