Skip to content

Simplify parser options passing, share ASTs even more #1158

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 28 commits into from
Jun 17, 2025
Merged
Show file tree
Hide file tree
Changes from 23 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
14e7b32
Create SourceFileParseOptions
jakebailey Jun 10, 2025
db149ad
Fix parsing mode
jakebailey Jun 10, 2025
d8c50f8
Script kind maybe
jakebailey Jun 10, 2025
b795913
partial
jakebailey Jun 10, 2025
10aa6ee
Missed one
jakebailey Jun 10, 2025
73ec3d9
Consolidate JSON parsing
jakebailey Jun 10, 2025
adfea3b
Tweak test
jakebailey Jun 10, 2025
c6b2b07
Binder doesn't need options anymore
jakebailey Jun 10, 2025
c3144c4
lint
jakebailey Jun 10, 2025
12f1795
Don't set ExternalModuleIndicator on JSON files
jakebailey Jun 10, 2025
a127ce6
Refactor a bit
jakebailey Jun 10, 2025
251e04a
Fix generator
jakebailey Jun 10, 2025
4cb0223
Remove TODOs
jakebailey Jun 10, 2025
a9bf41c
Shift code around, remove dead
jakebailey Jun 10, 2025
636eaf0
fmt
jakebailey Jun 11, 2025
b786e30
Merge branch 'main' into jabaile/external-module-indicator-parse-options
jakebailey Jun 11, 2025
f647de4
Merge branch 'main' into jabaile/external-module-indicator-parse-options
jakebailey Jun 16, 2025
17da1a7
Merge branch 'main' into jabaile/external-module-indicator-parse-options
jakebailey Jun 16, 2025
645fa76
value type
jakebailey Jun 16, 2025
60f6c21
Shift everything around a bunch, new finer grained key
jakebailey Jun 17, 2025
7783c4a
TODOs
jakebailey Jun 17, 2025
64ec265
Merge branch 'main' into jabaile/external-module-indicator-parse-options
jakebailey Jun 17, 2025
bc84ad6
Condense key down even more, awesome
jakebailey Jun 17, 2025
02183a2
No source file affecting for declaration files, amazing
jakebailey Jun 17, 2025
fa9628c
Move code around
jakebailey Jun 17, 2025
f7e3067
fmt
jakebailey Jun 17, 2025
9f01650
move again
jakebailey Jun 17, 2025
64249aa
Restore fileName for debugging
jakebailey Jun 17, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions internal/api/encoder/encoder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,21 @@ import (
"github.com/microsoft/typescript-go/internal/core"
"github.com/microsoft/typescript-go/internal/parser"
"github.com/microsoft/typescript-go/internal/repo"
"github.com/microsoft/typescript-go/internal/scanner"
"github.com/microsoft/typescript-go/internal/testutil/baseline"
"gotest.tools/v3/assert"
)

var parseCompilerOptions = &core.SourceFileAffectingCompilerOptions{
var parseCompilerOptions = core.SourceFileAffectingCompilerOptions{
EmitScriptTarget: core.ScriptTargetLatest,
}

func TestEncodeSourceFile(t *testing.T) {
t.Parallel()
sourceFile := parser.ParseSourceFile("/test.ts", "/test.ts", "import { bar } from \"bar\";\nexport function foo<T, U>(a: string, b: string): any {}\nfoo();", parseCompilerOptions, nil, scanner.JSDocParsingModeParseAll)
sourceFile := parser.ParseSourceFile(ast.SourceFileParseOptions{
FileName: "/test.ts",
Path: "/test.ts",
CompilerOptions: parseCompilerOptions,
}, "import { bar } from \"bar\";\nexport function foo<T, U>(a: string, b: string): any {}\nfoo();", core.ScriptKindTS)
t.Run("baseline", func(t *testing.T) {
t.Parallel()
buf, err := encoder.EncodeSourceFile(sourceFile, "")
Expand All @@ -42,14 +45,11 @@ func BenchmarkEncodeSourceFile(b *testing.B) {
filePath := filepath.Join(repo.TypeScriptSubmodulePath, "src/compiler/checker.ts")
fileContent, err := os.ReadFile(filePath)
assert.NilError(b, err)
sourceFile := parser.ParseSourceFile(
"/checker.ts",
"/checker.ts",
string(fileContent),
parseCompilerOptions,
nil,
scanner.JSDocParsingModeParseAll,
)
sourceFile := parser.ParseSourceFile(ast.SourceFileParseOptions{
FileName: "/checker.ts",
Path: "/checker.ts",
CompilerOptions: parseCompilerOptions,
}, string(fileContent), core.ScriptKindTS)

for b.Loop() {
_, err := encoder.EncodeSourceFile(sourceFile, "")
Expand Down
55 changes: 36 additions & 19 deletions internal/ast/ast.go
Original file line number Diff line number Diff line change
Expand Up @@ -9962,6 +9962,23 @@ type CommentDirective struct {

// SourceFile

type JSDocParsingMode int

const (
JSDocParsingModeParseAll JSDocParsingMode = iota
JSDocParsingModeParseNone
JSDocParsingModeParseForTypeErrors
JSDocParsingModeParseForTypeInfo
)

type SourceFileParseOptions struct {
FileName string
Path tspath.Path
CompilerOptions core.SourceFileAffectingCompilerOptions
ExternalModuleIndicatorOptions ExternalModuleIndicatorOptions
JSDocParsingMode JSDocParsingMode
}

type SourceFileMetaData struct {
PackageJsonType string
PackageJsonDirectory string
Expand All @@ -9985,17 +10002,13 @@ type SourceFile struct {
compositeNodeBase

// Fields set by NewSourceFile

text string
fileName string
path tspath.Path
Statements *NodeList // NodeList[*Statement]
parseOptions SourceFileParseOptions
text string
Statements *NodeList // NodeList[*Statement]

// Fields set by parser

diagnostics []*Diagnostic
jsdocDiagnostics []*Diagnostic
LanguageVersion core.ScriptTarget
LanguageVariant core.LanguageVariant
ScriptKind core.ScriptKind
IsDeclarationFile bool
Expand Down Expand Up @@ -10040,30 +10053,35 @@ type SourceFile struct {
tokenCache map[core.TextRange]*Node
}

func (f *NodeFactory) NewSourceFile(text string, fileName string, path tspath.Path, statements *NodeList) *Node {
if (tspath.GetEncodedRootLength(fileName) == 0 && !strings.HasPrefix(fileName, "^/")) || fileName != tspath.NormalizePath(fileName) {
panic(fmt.Sprintf("fileName should be normalized and absolute: %q", fileName))
func (f *NodeFactory) NewSourceFile(opts SourceFileParseOptions, text string, statements *NodeList) *Node {
if (tspath.GetEncodedRootLength(opts.FileName) == 0 && !strings.HasPrefix(opts.FileName, "^/")) || opts.FileName != tspath.NormalizePath(opts.FileName) {
panic(fmt.Sprintf("fileName should be normalized and absolute: %q", opts.FileName))
}

data := &SourceFile{}
data.parseOptions = opts
data.text = text
data.fileName = fileName
data.path = path
data.Statements = statements
data.LanguageVersion = core.ScriptTargetLatest
return f.newNode(KindSourceFile, data)
}

func (node *SourceFile) ParseOptions() SourceFileParseOptions {
return node.parseOptions
}

func (node *SourceFile) LanguageVersion() core.ScriptTarget {
return node.parseOptions.CompilerOptions.EmitScriptTarget
}

func (node *SourceFile) Text() string {
return node.text
}

func (node *SourceFile) FileName() string {
return node.fileName
return node.parseOptions.FileName
}

func (node *SourceFile) Path() tspath.Path {
return node.path
return node.parseOptions.Path
}

func (node *SourceFile) OriginalFileName() string {
Expand Down Expand Up @@ -10120,7 +10138,6 @@ func (node *SourceFile) IsJS() bool {

func (node *SourceFile) copyFrom(other *SourceFile) {
// Do not copy fields set by NewSourceFile (Text, FileName, Path, or Statements)
node.LanguageVersion = other.LanguageVersion
node.LanguageVariant = other.LanguageVariant
node.ScriptKind = other.ScriptKind
node.IsDeclarationFile = other.IsDeclarationFile
Expand All @@ -10141,7 +10158,7 @@ func (node *SourceFile) copyFrom(other *SourceFile) {
}

func (node *SourceFile) Clone(f NodeFactoryCoercible) *Node {
updated := f.AsNodeFactory().NewSourceFile(node.Text(), node.FileName(), node.Path(), node.Statements)
updated := f.AsNodeFactory().NewSourceFile(node.parseOptions, node.text, node.Statements)
newFile := updated.AsSourceFile()
newFile.copyFrom(node)
return cloneNode(updated, node.AsNode(), f.AsNodeFactory().hooks)
Expand All @@ -10153,7 +10170,7 @@ func (node *SourceFile) computeSubtreeFacts() SubtreeFacts {

func (f *NodeFactory) UpdateSourceFile(node *SourceFile, statements *StatementList) *Node {
if statements != node.Statements {
updated := f.NewSourceFile(node.Text(), node.fileName, node.path, statements).AsSourceFile()
updated := f.NewSourceFile(node.parseOptions, node.text, statements).AsSourceFile()
updated.copyFrom(node)
return updateNode(updated.AsNode(), node.AsNode(), f.hooks)
}
Expand Down
137 changes: 137 additions & 0 deletions internal/ast/externalmoduleindicator.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
package ast

import (
"github.com/microsoft/typescript-go/internal/core"
"github.com/microsoft/typescript-go/internal/tspath"
)

type ExternalModuleIndicatorOptions struct {
jsx bool
force bool
}
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shifting this code around now means that the entire cache key for the external module indicator is just these two booleans! So much more reuse.


func GetExternalModuleIndicatorOptions(fileName string, options *core.CompilerOptions, metadata SourceFileMetaData) ExternalModuleIndicatorOptions {
if tspath.IsDeclarationFileName(fileName) {
return ExternalModuleIndicatorOptions{}
}

switch options.GetEmitModuleDetectionKind() {
case core.ModuleDetectionKindForce:
// All non-declaration files are modules, declaration files still do the usual isFileProbablyExternalModule
return ExternalModuleIndicatorOptions{force: true}
case core.ModuleDetectionKindLegacy:
// Files are modules if they have imports, exports, or import.meta
return ExternalModuleIndicatorOptions{}
case core.ModuleDetectionKindAuto:
// If module is nodenext or node16, all esm format files are modules
// If jsx is react-jsx or react-jsxdev then jsx tags force module-ness
// otherwise, the presence of import or export statments (or import.meta) implies module-ness
return ExternalModuleIndicatorOptions{
jsx: options.Jsx == core.JsxEmitReactJSX || options.Jsx == core.JsxEmitReactJSXDev,
force: isFileForcedToBeModuleByFormat(fileName, options, metadata),
}
default:
return ExternalModuleIndicatorOptions{}
}
}

var isFileForcedToBeModuleByFormatExtensions = []string{tspath.ExtensionCjs, tspath.ExtensionCts, tspath.ExtensionMjs, tspath.ExtensionMts}

func isFileForcedToBeModuleByFormat(fileName string, options *core.CompilerOptions, metadata SourceFileMetaData) bool {
// Excludes declaration files - they still require an explicit `export {}` or the like
// for back compat purposes. The only non-declaration files _not_ forced to be a module are `.js` files
// that aren't esm-mode (meaning not in a `type: module` scope).
if GetImpliedNodeFormatForEmitWorker(fileName, options.GetEmitModuleKind(), metadata) == core.ModuleKindESNext || tspath.FileExtensionIsOneOf(fileName, isFileForcedToBeModuleByFormatExtensions) {
return true
}
return false
}

func SetExternalModuleIndicator(file *SourceFile, opts ExternalModuleIndicatorOptions) {
file.ExternalModuleIndicator = getExternalModuleIndicator(file, opts)
}

func getExternalModuleIndicator(file *SourceFile, opts ExternalModuleIndicatorOptions) *Node {
if file.ScriptKind == core.ScriptKindJSON {
return nil
}

if node := isFileProbablyExternalModule(file); node != nil {
return node
}

if file.IsDeclarationFile {
return nil
}

if opts.jsx {
if node := isFileModuleFromUsingJSXTag(file); node != nil {
return node
}
}

if opts.force {
return file.AsNode()
}

return nil
}

func isFileProbablyExternalModule(sourceFile *SourceFile) *Node {
for _, statement := range sourceFile.Statements.Nodes {
if IsExternalModuleIndicator(statement) {
return statement
}
}
return getImportMetaIfNecessary(sourceFile)
}

func getImportMetaIfNecessary(sourceFile *SourceFile) *Node {
if sourceFile.AsNode().Flags&NodeFlagsPossiblyContainsImportMeta != 0 {
return findChildNode(sourceFile.AsNode(), IsImportMeta)
}
return nil
}

func findChildNode(root *Node, check func(*Node) bool) *Node {
var result *Node
var visit func(*Node) bool
visit = func(node *Node) bool {
if check(node) {
result = node
return true
}
return node.ForEachChild(visit)
}
visit(root)
return result
}

func isFileModuleFromUsingJSXTag(file *SourceFile) *Node {
return walkTreeForJSXTags(file.AsNode())
}

// This is a somewhat unavoidable full tree walk to locate a JSX tag - `import.meta` requires the same,
// but we avoid that walk (or parts of it) if at all possible using the `PossiblyContainsImportMeta` node flag.
// Unfortunately, there's no `NodeFlag` space to do the same for JSX.
func walkTreeForJSXTags(node *Node) *Node {
var found *Node

var visitor func(node *Node) bool
visitor = func(node *Node) bool {
if found != nil {
return true
}
if node.SubtreeFacts()&SubtreeContainsJsx == 0 {
return false
}
if IsJsxOpeningElement(node) || IsJsxFragment(node) {
found = node
return true
}
return node.ForEachChild(visitor)
}
visitor(node)

return found
}
20 changes: 4 additions & 16 deletions internal/ast/utilities.go
Original file line number Diff line number Diff line change
Expand Up @@ -2549,27 +2549,24 @@ func GetImpliedNodeFormatForFile(path string, packageJsonType string) core.Modul
return impliedNodeFormat
}

func GetEmitModuleFormatOfFileWorker(fileName string, options *core.CompilerOptions, sourceFileMetaData *SourceFileMetaData) core.ModuleKind {
func GetEmitModuleFormatOfFileWorker(fileName string, options *core.CompilerOptions, sourceFileMetaData SourceFileMetaData) core.ModuleKind {
result := GetImpliedNodeFormatForEmitWorker(fileName, options.GetEmitModuleKind(), sourceFileMetaData)
if result != core.ModuleKindNone {
return result
}
return options.GetEmitModuleKind()
}

func GetImpliedNodeFormatForEmitWorker(fileName string, emitModuleKind core.ModuleKind, sourceFileMetaData *SourceFileMetaData) core.ResolutionMode {
func GetImpliedNodeFormatForEmitWorker(fileName string, emitModuleKind core.ModuleKind, sourceFileMetaData SourceFileMetaData) core.ResolutionMode {
if core.ModuleKindNode16 <= emitModuleKind && emitModuleKind <= core.ModuleKindNodeNext {
if sourceFileMetaData == nil {
return core.ModuleKindNone
}
return sourceFileMetaData.ImpliedNodeFormat
}
if sourceFileMetaData != nil && sourceFileMetaData.ImpliedNodeFormat == core.ModuleKindCommonJS &&
if sourceFileMetaData.ImpliedNodeFormat == core.ModuleKindCommonJS &&
(sourceFileMetaData.PackageJsonType == "commonjs" ||
tspath.FileExtensionIsOneOf(fileName, []string{tspath.ExtensionCjs, tspath.ExtensionCts})) {
return core.ModuleKindCommonJS
}
if sourceFileMetaData != nil && sourceFileMetaData.ImpliedNodeFormat == core.ModuleKindESNext &&
if sourceFileMetaData.ImpliedNodeFormat == core.ModuleKindESNext &&
(sourceFileMetaData.PackageJsonType == "module" ||
tspath.FileExtensionIsOneOf(fileName, []string{tspath.ExtensionMjs, tspath.ExtensionMts})) {
return core.ModuleKindESNext
Expand Down Expand Up @@ -2899,15 +2896,6 @@ func GetClassLikeDeclarationOfSymbol(symbol *Symbol) *Node {
return core.Find(symbol.Declarations, IsClassLike)
}

func GetLanguageVariant(scriptKind core.ScriptKind) core.LanguageVariant {
switch scriptKind {
case core.ScriptKindTSX, core.ScriptKindJSX, core.ScriptKindJS, core.ScriptKindJSON:
// .tsx and .jsx files are treated as jsx language variant.
return core.LanguageVariantJSX
}
return core.LanguageVariantStandard
}

func IsCallLikeExpression(node *Node) bool {
switch node.Kind {
case KindJsxOpeningElement, KindJsxSelfClosingElement, KindJsxOpeningFragment, KindCallExpression, KindNewExpression,
Expand Down
Loading