From 142bcb71f9dd7f0ea2cf186608462673743f94f4 Mon Sep 17 00:00:00 2001 From: Nate Moore Date: Wed, 6 Jul 2022 17:03:10 -0500 Subject: [PATCH 1/2] fix: handle nested expressions edge case --- .changeset/eight-teachers-sniff.md | 5 +++++ internal/printer/printer_test.go | 21 +++++++++++++++++++ internal/token.go | 33 ++++++++++++++++++++++++++---- internal/token_test.go | 14 +++++++++++++ 4 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 .changeset/eight-teachers-sniff.md diff --git a/.changeset/eight-teachers-sniff.md b/.changeset/eight-teachers-sniff.md new file mode 100644 index 000000000..ac6b9ae77 --- /dev/null +++ b/.changeset/eight-teachers-sniff.md @@ -0,0 +1,5 @@ +--- +'@astrojs/compiler': patch +--- + +Properly handle nested expressions that return multiple elements diff --git a/internal/printer/printer_test.go b/internal/printer/printer_test.go index ac3cde831..c3c5cd02a 100644 --- a/internal/printer/printer_test.go +++ b/internal/printer/printer_test.go @@ -619,6 +619,27 @@ const groups = [[0, 1, 2], [3, 4, 5]]; code: `${$$maybeRenderHead($$result)}
${(previous || next) && $$render` + BACKTICK + `` + BACKTICK + `}
`, }, }, + { + name: "nested expressions II", + source: `
{(previous || next) && }
`, + want: want{ + code: `${$$maybeRenderHead($$result)}
${(previous || next) && $$render` + BACKTICK + `` + BACKTICK + `}
`, + }, + }, + { + name: "nested expressions III", + source: `
{x.map((x) => x ?
{true ? {x} : null}
:
{false ? null : {x}}
)}
`, + want: want{ + code: "${$$maybeRenderHead($$result)}
${x.map((x) => x ? $$render`
${true ? $$render`${x}` : null}
` : $$render`
${false ? null : $$render`${x}`}
`)}
", + }, + }, + { + name: "nested expressions IV", + source: `
{() => { if (value > 0.25) { return Default } else if (value > 0.5) { return Another } else if (value > 0.75) { return Other } return Yet Other }}
`, + want: want{ + code: "${$$maybeRenderHead($$result)}
${() => { if (value > 0.25) { return $$render`Default`} else if (value > 0.5) { return $$render`Another`} else if (value > 0.75) { return $$render`Other`} return $$render`Yet Other`}}
", + }, + }, { name: "expressions with JS comments", source: `--- diff --git a/internal/token.go b/internal/token.go index 8bfe7b4dc..59b360c80 100644 --- a/internal/token.go +++ b/internal/token.go @@ -250,6 +250,7 @@ type Tokenizer struct { // tt is the TokenType of the current token. tt TokenType prevTokenType TokenType + prevTokens []Token fm FrontmatterState m MarkdownState // err is the first error encountered during tokenization. It is possible @@ -1331,23 +1332,47 @@ func (z *Tokenizer) isAtExpressionBoundary() bool { return true } +func (z *Tokenizer) trackPreviousTokens() { + if z.tt == StartExpressionToken { + z.prevTokens = make([]Token, 0) + } + if z.tt == StartTagToken { + z.prevTokens = append(z.prevTokens, z.Token()) + } else if z.tt == EndTagToken { + if len(z.prevTokens) > 0 { + for i := 1; i < len(z.prevTokens)+1; i++ { + tok := z.prevTokens[len(z.prevTokens)-i] + if tok.Data == string(z.buf[z.data.Start:z.data.End]) { + if len(z.prevTokens) == 1 { + z.prevTokens = make([]Token, 0) + } else { + z.prevTokens = z.prevTokens[0:i] + z.prevTokens = append(z.prevTokens, z.prevTokens[i:]...) + } + } + } + } + } +} + // Next scans the next token and returns its type. func (z *Tokenizer) Next() TokenType { + z.trackPreviousTokens() z.raw.Start = z.raw.End z.data.Start = z.raw.End z.data.End = z.raw.End z.prevTokenType = z.tt - // This handles expressions nested inside of Frontmatter elements - // but preserves `{}` as text outside of elements - if z.fm == FrontmatterOpen { + // Properly handle multiple nested expressions + if len(z.expressionStack) > 0 && len(z.prevTokens) == 0 { tt := z.Token().Type switch tt { - case StartTagToken, EndTagToken: + case StartTagToken, EndExpressionToken, TextToken: default: z.openBraceIsExpressionStart = false } } + if z.rawTag != "" { if z.rawTag == "plaintext" { // Read everything up to EOF. diff --git a/internal/token_test.go b/internal/token_test.go index 1a5e035a5..1f1cd8f03 100644 --- a/internal/token_test.go +++ b/internal/token_test.go @@ -198,6 +198,20 @@ func TestBasic(t *testing.T) { }}`, []TokenType{StartTagToken, StartExpressionToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, EndExpressionToken, EndTagToken}, }, + { + "expression with multiple elements", + `
{() => { + if (value > 0.25) { + return Default + } else if (value > 0.5) { + return Another + } else if (value > 0.75) { + return Other + } + return Yet Other + }}
`, + []TokenType{StartTagToken, StartExpressionToken, TextToken, TextToken, TextToken, TextToken, TextToken, StartTagToken, TextToken, EndTagToken, TextToken, TextToken, TextToken, TextToken, StartTagToken, TextToken, EndTagToken, TextToken, TextToken, TextToken, TextToken, StartTagToken, TextToken, EndTagToken, TextToken, TextToken, StartTagToken, TextToken, EndTagToken, TextToken, TextToken, EndExpressionToken, EndTagToken}, + }, { "attribute expression with quoted braces", `
`, From 071b78334651981e1584fb23d126d0022d6b7796 Mon Sep 17 00:00:00 2001 From: Nate Moore Date: Wed, 6 Jul 2022 17:22:17 -0500 Subject: [PATCH 2/2] chore: explain how we use trackPreviousTokens --- internal/token.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/token.go b/internal/token.go index 59b360c80..15cf0d20e 100644 --- a/internal/token.go +++ b/internal/token.go @@ -1333,13 +1333,17 @@ func (z *Tokenizer) isAtExpressionBoundary() bool { } func (z *Tokenizer) trackPreviousTokens() { - if z.tt == StartExpressionToken { + // Reset stack on expression boundaries + if z.tt == StartExpressionToken || z.tt == EndExpressionToken { z.prevTokens = make([]Token, 0) } if z.tt == StartTagToken { z.prevTokens = append(z.prevTokens, z.Token()) } else if z.tt == EndTagToken { if len(z.prevTokens) > 0 { + // This is a very simple stack that pops matching closing elements off the stack, + // which is good enough for our purposes. + // We only use this to track when `{` should be `StartExpressionToken` or `TextToken` for i := 1; i < len(z.prevTokens)+1; i++ { tok := z.prevTokens[len(z.prevTokens)-i] if tok.Data == string(z.buf[z.data.Start:z.data.End]) {