cue: remove all support for quoted identifiers

Note that QuoteIdent has now been removed alltogether.

Affected packages:
- cue/scanner
- cue/parser
- cue/ast
- tools/fix
- faulty spec reference

Fixes #1475

Signed-off-by: Marcel van Lohuizen <mpvl@golang.org>

Change-Id: Ide35cb383bc303955eef516928a7e1e5d0602fc7
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/531206
Unity-Result: CUEcueckoo <cueckoo@cuelang.org>
TryBot-Result: CUEcueckoo <cueckoo@cuelang.org>
Reviewed-by: Paul Jolly <paul@myitcv.io>

Author: mpvl

cue/ast/ident.go

@@ -68,38 +68,6 @@ func IsValidIdent(ident string) bool {
 	return true
 }
 
-// QuoteIdent quotes an identifier, if needed, and reports
-// an error if the identifier is invalid.
-//
-// Deprecated: quoted identifiers are deprecated. Use aliases.
-func QuoteIdent(ident string) (string, error) {
-	if ident != "" && ident[0] == '`' {
-		if _, err := strconv.Unquote(ident); err != nil {
-			return "", errors.Newf(token.NoPos, "invalid quoted identifier %q", ident)
-		}
-		return ident, nil
-	}
-
-	// TODO: consider quoting keywords
-	// switch ident {
-	// case "for", "in", "if", "let", "true", "false", "null":
-	// 	goto escape
-	// }
-
-	for _, r := range ident {
-		if isLetter(r) || isDigit(r) || r == '_' || r == '$' {
-			continue
-		}
-		if r == '-' {
-			return "`" + ident + "`", nil
-		}
-		return "", errors.Newf(token.NoPos, "invalid character '%s' in identifier", string(r))
-	}
-
-	_, err := parseIdent(token.NoPos, ident)
-	return ident, err
-}
-
 // ParseIdent unquotes a possibly quoted identifier and validates
 // if the result is valid.
 //

cue/parser/parser.go

@@ -388,10 +388,6 @@ func (p *parser) next() {
 			p.comments.add(comment)
 		}
 	}
-
-	if p.tok == token.IDENT && p.lit[0] == '`' {
-		p.assertV0(p.pos, 0, 13, "quoted identifiers")
-	}
 }
 
 // assertV0 indicates the last version at which a certain feature was

cue/scanner/scanner.go

@@ -296,32 +296,6 @@ func isExtendedIdent(r rune) bool {
 	return strings.IndexRune("-_#$%. ", r) >= 0
 }
 
-func (s *Scanner) scanQuotedIdentifier() string {
-	offs := s.offset - 1 // quote already consumed
-	hasInvalid := false
-	for ; ; s.next() {
-		switch {
-		default:
-			if !hasInvalid {
-				s.errf(s.offset, "invalid character '%s' in identifier", string(s.ch))
-				hasInvalid = true
-			}
-			continue
-
-		case isLetter(s.ch) || isDigit(s.ch) || isExtendedIdent(s.ch):
-			continue
-
-		case s.ch == '`':
-			s.next()
-			return string(s.src[offs:s.offset])
-
-		case s.ch == '\n':
-			s.errf(s.offset, "quoted identifier not terminated")
-			return string(s.src[offs:s.offset])
-		}
-	}
-}
-
 func digitVal(ch rune) int {
 	switch {
 	case '0' <= ch && ch <= '9':
@@ -821,10 +795,6 @@ scanAgain:
 				lit = "_" + s.scanFieldIdentifier()
 			}
 			insertEOL = true
-		case '`':
-			tok = token.IDENT
-			lit = s.scanQuotedIdentifier()
-			insertEOL = true
 
 		case '\n':
 			// we only reach here if s.insertComma was

cue/scanner/scanner_test.go

@@ -85,7 +85,6 @@ var testTokens = [...]elt{
 	{token.IDENT, "#_foobar", literal},
 	{token.IDENT, "_#foobar", literal},
 	{token.IDENT, "__#foobar", literal},
-	{token.IDENT, "`foobar`", literal},
 	{token.IDENT, "a۰۱۸", literal},
 	{token.IDENT, "foo६४", literal},
 	{token.IDENT, "bar９８７６", literal},
@@ -739,14 +738,13 @@ var errorTests = []struct {
 	lit string
 	err string
 }{
+	{"`", token.ILLEGAL, 0, "", "illegal character U+0060 '`'"},
+
 	{"\a", token.ILLEGAL, 0, "", "illegal character U+0007"},
 	{`^`, token.ILLEGAL, 0, "", "illegal character U+005E '^'"},
 	{`…`, token.ILLEGAL, 0, "", "illegal character U+2026 '…'"},
 	{`_|`, token.ILLEGAL, 0, "", "illegal token '_|'; expected '_'"},
 
-	{"`foo=bar`", token.IDENT, 4, "`foo=bar`", "invalid character '=' in identifier"},
-	{"`foo\nbar`", token.IDENT, 4, "`foo", "quoted identifier not terminated"},
-
 	{`@`, token.ATTRIBUTE, 1, `@`, "invalid attribute: expected '('"},
 	{`@foo`, token.ATTRIBUTE, 4, `@foo`, "invalid attribute: expected '('"},
 	{`@foo(`, token.ATTRIBUTE, 5, `@foo(`, "attribute missing ')'"},
@@ -784,6 +782,7 @@ var errorTests = []struct {
 	{`"\U00000000"`, token.STRING, 0, `"\U00000000"`, ""},
 	{`"\Uffffffff"`, token.STRING, 2, `"\Uffffffff"`, "escape sequence is invalid Unicode code point"},
 	{`'`, token.STRING, 0, `'`, "string literal not terminated"},
+	{`"`, token.STRING, 0, `"`, "string literal not terminated"},
 	{`""`, token.STRING, 0, `""`, ""},
 	{`"abc`, token.STRING, 0, `"abc`, "string literal not terminated"},
 	{`""abc`, token.STRING, 0, `""`, ""},

cue/testdata/resolve/003_resolution_of_quoted_identifiers.txtar

@@ -484,7 +484,6 @@ These examples all represent the same string:
 ```
 "日本語"                                 // UTF-8 input text
 '日本語'                                 // UTF-8 input text as byte sequence
-`日本語`                                 // UTF-8 input text as a raw literal
 "\u65e5\u672c\u8a9e"                    // the explicit Unicode code points
 "\U000065e5\U0000672c\U00008a9e"        // the explicit Unicode code points
 '\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e'  // the explicit UTF-8 bytes

doc/ref/spec.md

@@ -20,7 +20,6 @@
 package fix
 
 import (
-	"fmt"
 	"strings"
 
 	"cuelang.org/go/cue/ast"
@@ -122,95 +121,6 @@ func File(f *ast.File, o ...Option) *ast.File {
 		return true
 	}, nil)
 
-	// Referred nodes and used identifiers.
-	referred := map[ast.Node]string{}
-	used := map[string]bool{}
-	replacement := map[ast.Node]string{}
-
-	ast.Walk(f, func(n ast.Node) bool {
-		if i, ok := n.(*ast.Ident); ok {
-			str, err := ast.ParseIdent(i)
-			if err != nil {
-				return false
-			}
-			referred[i.Node] = str
-			used[str] = true
-		}
-		return true
-	}, nil)
-
-	num := 0
-	newIdent := func() string {
-		for num++; ; num++ {
-			str := fmt.Sprintf("X%d", num)
-			if !used[str] {
-				used[str] = true
-				return str
-			}
-		}
-	}
-
-	// Rewrite quoted identifier fields that are referenced.
-	f = astutil.Apply(f, func(c astutil.Cursor) bool {
-		n := c.Node()
-		switch x := n.(type) {
-		case *ast.Field:
-			m, ok := referred[x.Value]
-			if !ok {
-				break
-			}
-
-			if b, ok := x.Label.(*ast.Ident); ok {
-				str, err := ast.ParseIdent(b)
-				var expr ast.Expr = b
-
-				switch {
-				case token.Lookup(str) != token.IDENT:
-					// quote keywords
-					expr = ast.NewString(b.Name)
-
-				case err != nil || str != m || str == b.Name:
-					return true
-
-				case ast.IsValidIdent(str):
-					x.Label = astutil.CopyMeta(ast.NewIdent(str), x.Label).(ast.Label)
-					return true
-				}
-
-				ident := newIdent()
-				replacement[x.Value] = ident
-				expr = &ast.Alias{Ident: ast.NewIdent(ident), Expr: expr}
-				ast.SetRelPos(x.Label, token.NoRelPos)
-				x.Label = astutil.CopyMeta(expr, x.Label).(ast.Label)
-			}
-		}
-		return true
-	}, nil).(*ast.File)
-
-	// Replace quoted references with their alias identifier.
-	astutil.Apply(f, func(c astutil.Cursor) bool {
-		n := c.Node()
-		switch x := n.(type) {
-		case *ast.Ident:
-			if r, ok := replacement[x.Node]; ok {
-				c.Replace(astutil.CopyMeta(ast.NewIdent(r), n))
-				break
-			}
-			str, err := ast.ParseIdent(x)
-			if err != nil || str == x.Name {
-				break
-			}
-			// Either the identifier is valid, in which can be replaced simply
-			// as here, or it is a complicated identifier and the original
-			// destination must have been quoted, in which case it is handled
-			// above.
-			if ast.IsValidIdent(str) && token.Lookup(str) == token.IDENT {
-				c.Replace(astutil.CopyMeta(ast.NewIdent(str), n))
-			}
-		}
-		return true
-	}, nil)
-
 	// TODO: we are probably reintroducing slices. Disable for now.
 	//
 	// Rewrite slice expression.

tools/fix/fix.go

@@ -42,34 +42,6 @@ a: __div(1, 2)
 b: __mod(3, 5)
 c: __quo(2, 9)
 d: __rem(1.0, 1.0) // pass on illegal values.
-`,
-	}, {
-		name: "referenced quoted fields",
-		in: `package foo
-
-a: {
-	fiz: 4
-	faz: ` + "`fiz`" + `
-
-	// biz
-	` + "`biz`" + `: 5 // biz
-	buz: ` + "`biz`" + `
-	in: 3
-	ref: ` + "`in`" + ` & x
-}
-`,
-		out: `package foo
-
-a: {
-	fiz: 4
-	faz: fiz
-
-	// biz
-	biz:     5 // biz
-	buz:     biz
-	X1="in": 3
-	ref:     X1 & x
-}
 `,
 	}, {
 		in: `