Skip to content

Commit

Permalink
make webpack magic comment parsing more robust
Browse files Browse the repository at this point in the history
  • Loading branch information
@evanw
evanw committed Dec 3, 2022
1 parent fcdc8eb commit fdf184e
Show file tree
Hide file tree
Showing 5 changed files with 90 additions and 42 deletions.
4 changes: 2 additions & 2 deletions internal/js_ast/js_ast.go
View file
Expand Up @@ -743,7 +743,7 @@ type EImportString struct {
// because esbuild is not Webpack. But we do preserve them since doing so is
// harmless, easy to maintain, and useful to people. See the Webpack docs for
// more info: https://webpack.js.org/api/module-methods/#magic-comments.
LeadingInteriorComments []Comment
WebpackComments []Comment

ImportRecordIndex uint32
}
Expand All @@ -753,7 +753,7 @@ type EImportCall struct {
OptionsOrNil Expr

// See the comment for this same field on "EImportString" for more information
LeadingInteriorComments []Comment
WebpackComments []Comment
}

type Stmt struct {
Expand Down
53 changes: 45 additions & 8 deletions internal/js_lexer/js_lexer.go
View file
Expand Up @@ -245,7 +245,8 @@ type MaybeSubstring struct {
}

type Lexer struct {
CommentsToPreserveBefore []js_ast.Comment
LegalCommentsBeforeToken []js_ast.Comment
WebpackComments *[]js_ast.Comment
AllOriginalComments []logger.Range
Identifier MaybeSubstring
log logger.Log
Expand Down Expand Up @@ -288,7 +289,6 @@ type Lexer struct {
ts config.TSOptions
HasNewlineBefore bool
HasPureCommentBefore bool
PreserveAllCommentsBefore bool
IsLegacyOctalLiteral bool
PrevTokenWasAwaitKeyword bool
rescanCloseBraceAsTemplateToken bool
Expand Down Expand Up @@ -1212,7 +1212,7 @@ func (lexer *Lexer) Next() {
lexer.HasNewlineBefore = lexer.end == 0
lexer.HasPureCommentBefore = false
lexer.PrevTokenWasAwaitKeyword = false
lexer.CommentsToPreserveBefore = nil
lexer.LegalCommentsBeforeToken = nil

for {
lexer.start = lexer.end
Expand Down Expand Up @@ -2746,10 +2746,19 @@ func scanForPragmaArg(kind pragmaArg, start int, pragma string, text string) (lo
}, true
}

func isUpperASCII(c byte) bool {
return c >= 'A' && c <= 'Z'
}

func isLetterASCII(c byte) bool {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}

func (lexer *Lexer) scanCommentText() {
text := lexer.source.Contents[lexer.start:lexer.end]
hasLegalAnnotation := len(text) > 2 && text[2] == '!'
isMultiLineComment := text[1] == '*'
isWebpackComment := false

// Save the original comment text so we can subtract comments from the
// character frequency analysis used by symbol minification
Expand Down Expand Up @@ -2800,15 +2809,43 @@ func (lexer *Lexer) scanCommentText() {
lexer.SourceMappingURL = arg
}
}

case 'w':
// Webpack magic comments use this regular expression: /(^|\W)webpack[A-Z]{1,}[A-Za-z]{1,}:/
if lexer.WebpackComments != nil && !isWebpackComment && strings.HasPrefix(text[i:], "webpack") && !isLetterASCII(text[i-1]) {
j := i + 7
upperCount := 0
for isUpperASCII(text[j]) {
upperCount++
j++
}
if upperCount > 0 {
letterCount := 0
for isLetterASCII(text[j]) {
letterCount++
j++
}
if letterCount > 0 && text[j] == ':' {
isWebpackComment = true
}
}
}
}
}

if hasLegalAnnotation || lexer.PreserveAllCommentsBefore {
if isMultiLineComment {
text = helpers.RemoveMultiLineCommentIndent(lexer.source.Contents[:lexer.start], text)
}
if isMultiLineComment && (hasLegalAnnotation || isWebpackComment) {
text = helpers.RemoveMultiLineCommentIndent(lexer.source.Contents[:lexer.start], text)
}

if hasLegalAnnotation {
lexer.LegalCommentsBeforeToken = append(lexer.LegalCommentsBeforeToken, js_ast.Comment{
Loc: logger.Loc{Start: int32(lexer.start)},
Text: text,
})
}

lexer.CommentsToPreserveBefore = append(lexer.CommentsToPreserveBefore, js_ast.Comment{
if isWebpackComment {
*lexer.WebpackComments = append(*lexer.WebpackComments, js_ast.Comment{
Loc: logger.Loc{Start: int32(lexer.start)},
Text: text,
})
Expand Down
25 changes: 13 additions & 12 deletions internal/js_parser/js_parser.go
View file
Expand Up @@ -3673,10 +3673,10 @@ func (p *parser) parseImportExpr(loc logger.Loc, level js_ast.L) js_ast.Expr {
oldAllowIn := p.allowIn
p.allowIn = true

p.lexer.PreserveAllCommentsBefore = true
var webpackComments []js_ast.Comment
oldWebpackComments := p.lexer.WebpackComments
p.lexer.WebpackComments = &webpackComments
p.lexer.Expect(js_lexer.TOpenParen)
comments := p.lexer.CommentsToPreserveBefore
p.lexer.PreserveAllCommentsBefore = false

value := p.parseExpr(js_ast.LComma)
var optionsOrNil js_ast.Expr
Expand All @@ -3696,13 +3696,14 @@ func (p *parser) parseImportExpr(loc logger.Loc, level js_ast.L) js_ast.Expr {
}
}

p.lexer.WebpackComments = oldWebpackComments
p.lexer.Expect(js_lexer.TCloseParen)

p.allowIn = oldAllowIn
return js_ast.Expr{Loc: loc, Data: &js_ast.EImportCall{
Expr: value,
OptionsOrNil: optionsOrNil,
LeadingInteriorComments: comments,
Expr: value,
OptionsOrNil: optionsOrNil,
WebpackComments: webpackComments,
}}
}

Expand Down Expand Up @@ -7256,7 +7257,7 @@ func (p *parser) parseStmtsUpTo(end js_lexer.T, opts parseStmtOpts) []js_ast.Stm

for {
// Preserve some statement-level comments
comments := p.lexer.CommentsToPreserveBefore
comments := p.lexer.LegalCommentsBeforeToken
if len(comments) > 0 {
for _, comment := range comments {
stmts = append(stmts, js_ast.Stmt{
Expand Down Expand Up @@ -14054,8 +14055,8 @@ func (p *parser) visitExprInOut(expr js_ast.Expr, in exprIn) (js_ast.Expr, exprO
}
p.importRecordsForCurrentPart = append(p.importRecordsForCurrentPart, importRecordIndex)
return js_ast.Expr{Loc: expr.Loc, Data: &js_ast.EImportString{
ImportRecordIndex: importRecordIndex,
LeadingInteriorComments: e.LeadingInteriorComments,
ImportRecordIndex: importRecordIndex,
WebpackComments: e.WebpackComments,
}}
}

Expand Down Expand Up @@ -14108,9 +14109,9 @@ func (p *parser) visitExprInOut(expr js_ast.Expr, in exprIn) (js_ast.Expr, exprO
}

return js_ast.Expr{Loc: expr.Loc, Data: &js_ast.EImportCall{
Expr: arg,
OptionsOrNil: e.OptionsOrNil,
LeadingInteriorComments: e.LeadingInteriorComments,
Expr: arg,
OptionsOrNil: e.OptionsOrNil,
WebpackComments: e.WebpackComments,
}}
}), exprOut{}

Expand Down
26 changes: 13 additions & 13 deletions internal/js_printer/js_printer.go
View file
Expand Up @@ -1090,7 +1090,7 @@ func (p *printer) printQuotedUTF16(data []uint16, allowBacktick bool) {

func (p *printer) printRequireOrImportExpr(
importRecordIndex uint32,
leadingInteriorComments []js_ast.Comment,
webpackComments []js_ast.Comment,
level js_ast.L,
flags printExprFlags,
) {
Expand Down Expand Up @@ -1175,10 +1175,10 @@ func (p *printer) printRequireOrImportExpr(
p.print("(")
defer p.print(")")
}
if len(leadingInteriorComments) > 0 {
if len(webpackComments) > 0 {
p.printNewline()
p.options.Indent++
for _, comment := range leadingInteriorComments {
for _, comment := range webpackComments {
p.printIndentedComment(comment.Text)
}
p.printIndent()
Expand All @@ -1188,7 +1188,7 @@ func (p *printer) printRequireOrImportExpr(
if !p.options.UnsupportedFeatures.Has(compat.DynamicImport) {
p.printImportCallAssertions(record.Assertions)
}
if len(leadingInteriorComments) > 0 {
if len(webpackComments) > 0 {
p.printNewline()
p.options.Indent--
p.printIndent()
Expand Down Expand Up @@ -1915,17 +1915,17 @@ func (p *printer) printExpr(expr js_ast.Expr, level js_ast.L, flags printExprFla
}

case *js_ast.EImportString:
var leadingInteriorComments []js_ast.Comment
var webpackComments []js_ast.Comment
if !p.options.MinifyWhitespace {
leadingInteriorComments = e.LeadingInteriorComments
webpackComments = e.WebpackComments
}
p.addSourceMapping(expr.Loc)
p.printRequireOrImportExpr(e.ImportRecordIndex, leadingInteriorComments, level, flags)
p.printRequireOrImportExpr(e.ImportRecordIndex, webpackComments, level, flags)

case *js_ast.EImportCall:
var leadingInteriorComments []js_ast.Comment
var webpackComments []js_ast.Comment
if !p.options.MinifyWhitespace {
leadingInteriorComments = e.LeadingInteriorComments
webpackComments = e.WebpackComments
}
wrap := level >= js_ast.LNew || (flags&forbidCall) != 0
if wrap {
Expand All @@ -1934,10 +1934,10 @@ func (p *printer) printExpr(expr js_ast.Expr, level js_ast.L, flags printExprFla
p.printSpaceBeforeIdentifier()
p.addSourceMapping(expr.Loc)
p.print("import(")
if len(leadingInteriorComments) > 0 {
if len(webpackComments) > 0 {
p.printNewline()
p.options.Indent++
for _, comment := range leadingInteriorComments {
for _, comment := range webpackComments {
p.printIndentedComment(comment.Text)
}
p.printIndent()
Expand All @@ -1947,7 +1947,7 @@ func (p *printer) printExpr(expr js_ast.Expr, level js_ast.L, flags printExprFla
// Just omit import assertions if they aren't supported
if e.OptionsOrNil.Data != nil && !p.options.UnsupportedFeatures.Has(compat.ImportAssertions) {
p.print(",")
if len(leadingInteriorComments) > 0 {
if len(webpackComments) > 0 {
p.printNewline()
p.printIndent()
} else {
Expand All @@ -1956,7 +1956,7 @@ func (p *printer) printExpr(expr js_ast.Expr, level js_ast.L, flags printExprFla
p.printExpr(e.OptionsOrNil, js_ast.LComma, 0)
}

if len(leadingInteriorComments) > 0 {
if len(webpackComments) > 0 {
p.printNewline()
p.options.Indent--
p.printIndent()
Expand Down
24 changes: 17 additions & 7 deletions internal/js_printer/js_printer_test.go
View file
Expand Up @@ -667,13 +667,23 @@ func TestPrivateIdentifiers(t *testing.T) {
func TestImport(t *testing.T) {
expectPrinted(t, "import('path');", "import(\"path\");\n") // The semicolon must not be a separate statement

// Test preservation of leading interior comments
expectPrinted(t, "import(// comment 1\n // comment 2\n 'path');", "import(\n // comment 1\n // comment 2\n \"path\"\n);\n")
expectPrinted(t, "import(// comment 1\n // comment 2\n 'path', {type: 'module'});", "import(\n // comment 1\n // comment 2\n \"path\",\n { type: \"module\" }\n);\n")
expectPrinted(t, "import(/* comment 1 */ /* comment 2 */ 'path');", "import(\n /* comment 1 */\n /* comment 2 */\n \"path\"\n);\n")
expectPrinted(t, "import(/* comment 1 */ /* comment 2 */ 'path', {type: 'module'});", "import(\n /* comment 1 */\n /* comment 2 */\n \"path\",\n { type: \"module\" }\n);\n")
expectPrinted(t, "import(\n /* multi\n * line\n * comment */ 'path');", "import(\n /* multi\n * line\n * comment */\n \"path\"\n);\n")
expectPrinted(t, "import(/* comment 1 */ 'path' /* comment 2 */);", "import(\n /* comment 1 */\n \"path\"\n);\n")
// Test preservation of Webpack-specific comments
expectPrinted(t, "import(// webpackFoo: 1\n // webpackBar: 2\n 'path');", "import(\n // webpackFoo: 1\n // webpackBar: 2\n \"path\"\n);\n")
expectPrinted(t, "import(// webpackFoo: 1\n // webpackBar: 2\n 'path', {type: 'module'});", "import(\n // webpackFoo: 1\n // webpackBar: 2\n \"path\",\n { type: \"module\" }\n);\n")
expectPrinted(t, "import(/* webpackFoo: 1 */ /* webpackBar: 2 */ 'path');", "import(\n /* webpackFoo: 1 */\n /* webpackBar: 2 */\n \"path\"\n);\n")
expectPrinted(t, "import(/* webpackFoo: 1 */ /* webpackBar: 2 */ 'path', {type: 'module'});", "import(\n /* webpackFoo: 1 */\n /* webpackBar: 2 */\n \"path\",\n { type: \"module\" }\n);\n")
expectPrinted(t, "import(\n /* multi\n * line\n * webpackBar: */ 'path');", "import(\n /* multi\n * line\n * webpackBar: */\n \"path\"\n);\n")
expectPrinted(t, "import(/* webpackFoo: 1 */ 'path' /* webpackBar:2 */);", "import(\n /* webpackFoo: 1 */\n /* webpackBar:2 */\n \"path\"\n);\n")
expectPrinted(t, "import(/* webpackFoo: 1 */ 'path' /* webpackBar:2 */ ,);", "import(\n /* webpackFoo: 1 */\n /* webpackBar:2 */\n \"path\"\n);\n")
expectPrinted(t, "import(/* webpackFoo: 1 */ 'path', /* webpackBar:2 */ );", "import(\n /* webpackFoo: 1 */\n /* webpackBar:2 */\n \"path\"\n);\n")
expectPrinted(t, "import(/* webpackFoo: 1 */ 'path', { type: 'module' } /* webpackBar:2 */ );", "import(\n /* webpackFoo: 1 */\n /* webpackBar:2 */\n \"path\",\n { type: \"module\" }\n);\n")
expectPrinted(t, "import(new URL('path', /* webpackFoo: these can go anywhere */ import.meta.url))", "import(\n /* webpackFoo: these can go anywhere */\n new URL(\"path\", import.meta.url)\n);\n")

// Other comments should not be preserved
expectPrinted(t, "import(// comment 1\n // comment 2\n 'path');", "import(\"path\");\n")
expectPrinted(t, "import(// comment 1\n // comment 2\n 'path', {type: 'module'});", "import(\"path\", { type: \"module\" });\n")
expectPrinted(t, "import(/* comment 1 */ /* comment 2 */ 'path');", "import(\"path\");\n")
expectPrinted(t, "import(/* comment 1 */ /* comment 2 */ 'path', {type: 'module'});", "import(\"path\", { type: \"module\" });\n")
}

func TestExportDefault(t *testing.T) {
Expand Down

0 comments on commit fdf184e

Please sign in to comment.