From 209415dd4da30c2ef027bd4bc9066f6f7d2e3d6e Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Tue, 8 Apr 2025 17:32:45 -0700 Subject: [PATCH 01/10] Combine fileloader results into one struct --- internal/compiler/fileloader.go | 27 ++++++++++++++++----------- internal/compiler/program.go | 11 ++++------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/internal/compiler/fileloader.go b/internal/compiler/fileloader.go index c0e2ac2bd7..2126dc1b49 100644 --- a/internal/compiler/fileloader.go +++ b/internal/compiler/fileloader.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/microsoft/typescript-go/internal/ast" + "github.com/microsoft/typescript-go/internal/collections" "github.com/microsoft/typescript-go/internal/compiler/module" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/tsoptions" @@ -19,9 +20,8 @@ type fileLoader struct { programOptions ProgramOptions compilerOptions *core.CompilerOptions - resolver *module.Resolver - resolvedModulesMutex sync.Mutex - resolvedModules map[tspath.Path]module.ModeAwareCache[*module.ResolvedModule] + resolver *module.Resolver + resolvedModules collections.SyncMap[tspath.Path, module.ModeAwareCache[*module.ResolvedModule]] sourceFileMetaDatasMutex sync.RWMutex sourceFileMetaDatas map[tspath.Path]*ast.SourceFileMetaData @@ -36,6 +36,12 @@ type fileLoader struct { supportedExtensions []string } +type processedFiles struct { + files []*ast.SourceFile + resolvedModules map[tspath.Path]module.ModeAwareCache[*module.ResolvedModule] + sourceFileMetaDatas map[tspath.Path]*ast.SourceFileMetaData +} + func processAllProgramFiles( host CompilerHost, programOptions ProgramOptions, @@ -43,7 +49,7 @@ func processAllProgramFiles( resolver *module.Resolver, rootFiles []string, libs []string, -) (files []*ast.SourceFile, resolvedModules map[tspath.Path]module.ModeAwareCache[*module.ResolvedModule], sourceFileMetaDatas map[tspath.Path]*ast.SourceFileMetaData) { +) processedFiles { supportedExtensions := tsoptions.GetSupportedExtensions(compilerOptions, nil /*extraFileExtensions*/) loader := fileLoader{ host: host, @@ -79,7 +85,11 @@ func processAllProgramFiles( } loader.sortLibs(libFiles) - return append(libFiles, files...), loader.resolvedModules, loader.sourceFileMetaDatas + return processedFiles{ + files: append(libFiles, files...), + resolvedModules: loader.resolvedModules.ToMap(), + sourceFileMetaDatas: loader.sourceFileMetaDatas, + } } func (p *fileLoader) addRootTasks(files []string, isLib bool) { @@ -277,12 +287,7 @@ func (p *fileLoader) resolveImportsAndModuleAugmentations(file *ast.SourceFile) resolutionsInFile := make(module.ModeAwareCache[*module.ResolvedModule], len(resolutions)) - p.resolvedModulesMutex.Lock() - defer p.resolvedModulesMutex.Unlock() - if p.resolvedModules == nil { - p.resolvedModules = make(map[tspath.Path]module.ModeAwareCache[*module.ResolvedModule]) - } - p.resolvedModules[file.Path()] = resolutionsInFile + p.resolvedModules.Store(file.Path(), resolutionsInFile) for i, resolution := range resolutions { resolvedFileName := resolution.ResolvedFileName diff --git a/internal/compiler/program.go b/internal/compiler/program.go index afa7ae60b8..1f0184b2af 100644 --- a/internal/compiler/program.go +++ b/internal/compiler/program.go @@ -41,15 +41,13 @@ type Program struct { currentDirectory string configFileParsingDiagnostics []*ast.Diagnostic - resolver *module.Resolver - resolvedModules map[tspath.Path]module.ModeAwareCache[*module.ResolvedModule] + resolver *module.Resolver comparePathsOptions tspath.ComparePathsOptions - files []*ast.SourceFile - filesByPath map[tspath.Path]*ast.SourceFile + processedFiles - sourceFileMetaDatas map[tspath.Path]*ast.SourceFileMetaData + filesByPath map[tspath.Path]*ast.SourceFile // The below settings are to track if a .js file should be add to the program if loaded via searching under node_modules. // This works as imported modules are discovered recursively in a depth first manner, specifically: @@ -75,7 +73,6 @@ func NewProgram(options ProgramOptions) *Program { p.programOptions = options p.compilerOptions = options.Options p.configFileParsingDiagnostics = slices.Clip(options.ConfigFileParsingDiagnostics) - p.sourceFileMetaDatas = make(map[tspath.Path]*ast.SourceFileMetaData) if p.compilerOptions == nil { p.compilerOptions = &core.CompilerOptions{} } @@ -153,7 +150,7 @@ func NewProgram(options ProgramOptions) *Program { } } - p.files, p.resolvedModules, p.sourceFileMetaDatas = processAllProgramFiles(p.host, p.programOptions, p.compilerOptions, p.resolver, rootFiles, libs) + p.processedFiles = processAllProgramFiles(p.host, p.programOptions, p.compilerOptions, p.resolver, rootFiles, libs) p.filesByPath = make(map[tspath.Path]*ast.SourceFile, len(p.files)) for _, file := range p.files { p.filesByPath[file.Path()] = file From 37a213f28604ac73f10649a4fc5958719005b95b Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Tue, 8 Apr 2025 17:46:48 -0700 Subject: [PATCH 02/10] Reorganize fileLoader to be clear what is locked and what is not --- internal/compiler/fileloader.go | 51 ++++++++++++++------------------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/internal/compiler/fileloader.go b/internal/compiler/fileloader.go index 2126dc1b49..3a6c582eda 100644 --- a/internal/compiler/fileloader.go +++ b/internal/compiler/fileloader.go @@ -16,24 +16,22 @@ import ( ) type fileLoader struct { - host CompilerHost - programOptions ProgramOptions - compilerOptions *core.CompilerOptions - - resolver *module.Resolver - resolvedModules collections.SyncMap[tspath.Path, module.ModeAwareCache[*module.ResolvedModule]] - - sourceFileMetaDatasMutex sync.RWMutex - sourceFileMetaDatas map[tspath.Path]*ast.SourceFileMetaData - - mu sync.Mutex - wg core.WorkGroup - tasksByFileName map[string]*parseTask - currentNodeModulesDepth int - defaultLibraryPath string - comparePathsOptions tspath.ComparePathsOptions - rootTasks []*parseTask - supportedExtensions []string + host CompilerHost + programOptions ProgramOptions + compilerOptions *core.CompilerOptions + resolver *module.Resolver + defaultLibraryPath string + comparePathsOptions tspath.ComparePathsOptions + wg core.WorkGroup + supportedExtensions []string + + rootTasks []*parseTask + + resolvedModules collections.SyncMap[tspath.Path, module.ModeAwareCache[*module.ResolvedModule]] + sourceFileMetaDatas collections.SyncMap[tspath.Path, *ast.SourceFileMetaData] + + tasksByFileNameMu sync.Mutex + tasksByFileName map[string]*parseTask } type processedFiles struct { @@ -88,7 +86,7 @@ func processAllProgramFiles( return processedFiles{ files: append(libFiles, files...), resolvedModules: loader.resolvedModules.ToMap(), - sourceFileMetaDatas: loader.sourceFileMetaDatas, + sourceFileMetaDatas: loader.sourceFileMetaDatas.ToMap(), } } @@ -121,8 +119,8 @@ func (p *fileLoader) addAutomaticTypeDirectiveTasks() { func (p *fileLoader) startTasks(tasks []*parseTask) { if len(tasks) > 0 { - p.mu.Lock() - defer p.mu.Unlock() + p.tasksByFileNameMu.Lock() + defer p.tasksByFileNameMu.Unlock() for i, task := range tasks { // dedup tasks to ensure correct file order, regardless of which task would be started first if existingTask, ok := p.tasksByFileName[task.normalizedFilePath]; ok { @@ -235,9 +233,7 @@ func (t *parseTask) start(loader *fileLoader) { } func (p *fileLoader) loadSourceFileMetaData(path tspath.Path) { - p.sourceFileMetaDatasMutex.RLock() - _, ok := p.sourceFileMetaDatas[path] - p.sourceFileMetaDatasMutex.RUnlock() + _, ok := p.sourceFileMetaDatas.Load(path) if ok { return } @@ -249,12 +245,7 @@ func (p *fileLoader) loadSourceFileMetaData(path tspath.Path) { ImpliedNodeFormat: impliedNodeFormat, } - p.sourceFileMetaDatasMutex.Lock() - defer p.sourceFileMetaDatasMutex.Unlock() - if p.sourceFileMetaDatas == nil { - p.sourceFileMetaDatas = make(map[tspath.Path]*ast.SourceFileMetaData) - } - p.sourceFileMetaDatas[path] = metadata + p.sourceFileMetaDatas.Store(path, metadata) } func (p *fileLoader) parseSourceFile(fileName string) *ast.SourceFile { From 4240c51d8e927c14e568e0ba5c966765c7794b9d Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Tue, 8 Apr 2025 17:53:39 -0700 Subject: [PATCH 03/10] Make tasksByFileName lock-free --- internal/compiler/fileloader.go | 45 +++++++++++++++------------------ 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/internal/compiler/fileloader.go b/internal/compiler/fileloader.go index 3a6c582eda..dea4cadb67 100644 --- a/internal/compiler/fileloader.go +++ b/internal/compiler/fileloader.go @@ -5,7 +5,6 @@ import ( "iter" "slices" "strings" - "sync" "github.com/microsoft/typescript-go/internal/ast" "github.com/microsoft/typescript-go/internal/collections" @@ -25,13 +24,11 @@ type fileLoader struct { wg core.WorkGroup supportedExtensions []string - rootTasks []*parseTask + tasksByFileName collections.SyncMap[string, *parseTask] + rootTasks []*parseTask resolvedModules collections.SyncMap[tspath.Path, module.ModeAwareCache[*module.ResolvedModule]] sourceFileMetaDatas collections.SyncMap[tspath.Path, *ast.SourceFileMetaData] - - tasksByFileNameMu sync.Mutex - tasksByFileName map[string]*parseTask } type processedFiles struct { @@ -54,7 +51,6 @@ func processAllProgramFiles( programOptions: programOptions, compilerOptions: compilerOptions, resolver: resolver, - tasksByFileName: make(map[string]*parseTask), defaultLibraryPath: tspath.GetNormalizedAbsolutePath(host.DefaultLibraryPath(), host.GetCurrentDirectory()), comparePathsOptions: tspath.ComparePathsOptions{ UseCaseSensitiveFileNames: host.FS().UseCaseSensitiveFileNames(), @@ -119,14 +115,12 @@ func (p *fileLoader) addAutomaticTypeDirectiveTasks() { func (p *fileLoader) startTasks(tasks []*parseTask) { if len(tasks) > 0 { - p.tasksByFileNameMu.Lock() - defer p.tasksByFileNameMu.Unlock() for i, task := range tasks { - // dedup tasks to ensure correct file order, regardless of which task would be started first - if existingTask, ok := p.tasksByFileName[task.normalizedFilePath]; ok { - tasks[i] = existingTask + task, loaded := p.tasksByFileName.LoadOrStore(task.normalizedFilePath, task) + if loaded { + // dedup tasks to ensure correct file order, regardless of which task would be started first + tasks[i] = task } else { - p.tasksByFileName[task.normalizedFilePath] = task task.start(p) } } @@ -135,26 +129,27 @@ func (p *fileLoader) startTasks(tasks []*parseTask) { func (p *fileLoader) collectTasks(tasks []*parseTask) iter.Seq[*parseTask] { return func(yield func(*parseTask) bool) { - p.collectTasksWorker(tasks, yield) + p.collectTasksWorker(tasks, core.Set[*parseTask]{}, yield) } } -func (p *fileLoader) collectTasksWorker(tasks []*parseTask, yield func(*parseTask) bool) bool { +func (p *fileLoader) collectTasksWorker(tasks []*parseTask, seen core.Set[*parseTask], yield func(*parseTask) bool) bool { for _, task := range tasks { - if _, ok := p.tasksByFileName[task.normalizedFilePath]; ok { - // ensure we only walk each task once - delete(p.tasksByFileName, task.normalizedFilePath) + // ensure we only walk each task once + if seen.Has(task) { + continue + } + seen.Add(task) - if len(task.subTasks) > 0 { - if !p.collectTasksWorker(task.subTasks, yield) { - return false - } + if len(task.subTasks) > 0 { + if !p.collectTasksWorker(task.subTasks, seen, yield) { + return false } + } - if task.file != nil { - if !yield(task) { - return false - } + if task.file != nil { + if !yield(task) { + return false } } } From 78deec52445cae060dc79dc9a544cab55adfb2e0 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Tue, 8 Apr 2025 18:17:42 -0700 Subject: [PATCH 04/10] bitten by my own lint rule --- internal/compiler/fileloader.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/compiler/fileloader.go b/internal/compiler/fileloader.go index dea4cadb67..d07cce0280 100644 --- a/internal/compiler/fileloader.go +++ b/internal/compiler/fileloader.go @@ -116,12 +116,12 @@ func (p *fileLoader) addAutomaticTypeDirectiveTasks() { func (p *fileLoader) startTasks(tasks []*parseTask) { if len(tasks) > 0 { for i, task := range tasks { - task, loaded := p.tasksByFileName.LoadOrStore(task.normalizedFilePath, task) + loadedTask, loaded := p.tasksByFileName.LoadOrStore(task.normalizedFilePath, task) if loaded { // dedup tasks to ensure correct file order, regardless of which task would be started first - tasks[i] = task + tasks[i] = loadedTask } else { - task.start(p) + loadedTask.start(p) } } } From 37b90535848b65d549e61f4f974df028590504f4 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Tue, 8 Apr 2025 18:17:55 -0700 Subject: [PATCH 05/10] Revert "bitten by my own lint rule" This reverts commit 78deec52445cae060dc79dc9a544cab55adfb2e0. --- internal/compiler/fileloader.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/compiler/fileloader.go b/internal/compiler/fileloader.go index d07cce0280..dea4cadb67 100644 --- a/internal/compiler/fileloader.go +++ b/internal/compiler/fileloader.go @@ -116,12 +116,12 @@ func (p *fileLoader) addAutomaticTypeDirectiveTasks() { func (p *fileLoader) startTasks(tasks []*parseTask) { if len(tasks) > 0 { for i, task := range tasks { - loadedTask, loaded := p.tasksByFileName.LoadOrStore(task.normalizedFilePath, task) + task, loaded := p.tasksByFileName.LoadOrStore(task.normalizedFilePath, task) if loaded { // dedup tasks to ensure correct file order, regardless of which task would be started first - tasks[i] = loadedTask + tasks[i] = task } else { - loadedTask.start(p) + task.start(p) } } } From 3d92b3a00cdb5722c78e04e14579ac72fe667e1e Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Tue, 8 Apr 2025 18:26:13 -0700 Subject: [PATCH 06/10] Reapply "bitten by my own lint rule" This reverts commit 37b90535848b65d549e61f4f974df028590504f4. --- internal/compiler/fileloader.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/compiler/fileloader.go b/internal/compiler/fileloader.go index dea4cadb67..d07cce0280 100644 --- a/internal/compiler/fileloader.go +++ b/internal/compiler/fileloader.go @@ -116,12 +116,12 @@ func (p *fileLoader) addAutomaticTypeDirectiveTasks() { func (p *fileLoader) startTasks(tasks []*parseTask) { if len(tasks) > 0 { for i, task := range tasks { - task, loaded := p.tasksByFileName.LoadOrStore(task.normalizedFilePath, task) + loadedTask, loaded := p.tasksByFileName.LoadOrStore(task.normalizedFilePath, task) if loaded { // dedup tasks to ensure correct file order, regardless of which task would be started first - tasks[i] = task + tasks[i] = loadedTask } else { - task.start(p) + loadedTask.start(p) } } } From ec2e850e8ddfea9ebf639382c5b980a0b7c3ab57 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Wed, 9 Apr 2025 10:11:06 -0700 Subject: [PATCH 07/10] Add NewProgram benchmark --- internal/compiler/program_test.go | 54 +++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/internal/compiler/program_test.go b/internal/compiler/program_test.go index aa5ebf177e..0379028c37 100644 --- a/internal/compiler/program_test.go +++ b/internal/compiler/program_test.go @@ -1,12 +1,16 @@ package compiler import ( + "path/filepath" "slices" "strings" "testing" "github.com/microsoft/typescript-go/internal/bundled" "github.com/microsoft/typescript-go/internal/core" + "github.com/microsoft/typescript-go/internal/repo" + "github.com/microsoft/typescript-go/internal/tspath" + "github.com/microsoft/typescript-go/internal/vfs/osvfs" "github.com/microsoft/typescript-go/internal/vfs/vfstest" "gotest.tools/v3/assert" ) @@ -242,3 +246,53 @@ func TestProgram(t *testing.T) { }) } } + +func BenchmarkNewProgram(b *testing.B) { + if !bundled.Embedded { + // Without embedding, we'd need to read all of the lib files out from disk into the MapFS. + // Just skip this for now. + b.Skip("bundled files are not embedded") + } + + for _, testCase := range programTestCases { + b.Run(testCase.testName, func(b *testing.B) { + fs := vfstest.FromMap[any](nil, false /*useCaseSensitiveFileNames*/) + fs = bundled.WrapFS(fs) + + for _, testFile := range testCase.files { + _ = fs.WriteFile(testFile.fileName, testFile.contents, false) + } + + opts := core.CompilerOptions{Target: testCase.target} + programOpts := ProgramOptions{ + RootFiles: []string{"c:/dev/src/index.ts"}, + Host: NewCompilerHost(&opts, "c:/dev/src", fs, bundled.LibPath()), + Options: &opts, + SingleThreaded: false, + } + + for b.Loop() { + NewProgram(programOpts) + } + }) + } + + b.Run("compiler", func(b *testing.B) { + repo.SkipIfNoTypeScriptSubmodule(b) + + compilerDir := tspath.NormalizeSlashes(filepath.Join(repo.TypeScriptSubmodulePath, "src", "compiler")) + + fs := osvfs.FS() + fs = bundled.WrapFS(fs) + + opts := ProgramOptions{ + ConfigFileName: tspath.CombinePaths(compilerDir, "tsconfig.json"), + Host: NewCompilerHost(nil, compilerDir, fs, bundled.LibPath()), + SingleThreaded: false, + } + + for b.Loop() { + NewProgram(opts) + } + }) +} From 14235ef8e39d56241d290232c1b046afc1f02671 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Wed, 9 Apr 2025 10:51:32 -0700 Subject: [PATCH 08/10] truly lock-free --- internal/compiler/fileloader.go | 44 ++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/internal/compiler/fileloader.go b/internal/compiler/fileloader.go index d07cce0280..f5a247bed0 100644 --- a/internal/compiler/fileloader.go +++ b/internal/compiler/fileloader.go @@ -27,8 +27,8 @@ type fileLoader struct { tasksByFileName collections.SyncMap[string, *parseTask] rootTasks []*parseTask - resolvedModules collections.SyncMap[tspath.Path, module.ModeAwareCache[*module.ResolvedModule]] - sourceFileMetaDatas collections.SyncMap[tspath.Path, *ast.SourceFileMetaData] + // fileCount atomic.Int32 + // libFileCount atomic.Int32 } type processedFiles struct { @@ -69,6 +69,10 @@ func processAllProgramFiles( loader.wg.RunAndWait() + size := loader.tasksByFileName.Size() + resolvedModules := make(map[tspath.Path]module.ModeAwareCache[*module.ResolvedModule], size) + sourceFileMetaDatas := make(map[tspath.Path]*ast.SourceFileMetaData, size) + files, libFiles := []*ast.SourceFile{}, []*ast.SourceFile{} for task := range loader.collectTasks(loader.rootTasks) { if task.isLib { @@ -76,13 +80,15 @@ func processAllProgramFiles( } else { files = append(files, task.file) } + resolvedModules[task.file.Path()] = task.resolutionsInFile + sourceFileMetaDatas[task.file.Path()] = task.metadata } loader.sortLibs(libFiles) return processedFiles{ files: append(libFiles, files...), - resolvedModules: loader.resolvedModules.ToMap(), - sourceFileMetaDatas: loader.sourceFileMetaDatas.ToMap(), + resolvedModules: resolvedModules, + sourceFileMetaDatas: sourceFileMetaDatas, } } @@ -187,6 +193,9 @@ type parseTask struct { file *ast.SourceFile isLib bool subTasks []*parseTask + + metadata *ast.SourceFileMetaData + resolutionsInFile module.ModeAwareCache[*module.ResolvedModule] } func (t *parseTask) start(loader *fileLoader) { @@ -218,35 +227,30 @@ func (t *parseTask) start(loader *fileLoader) { } } - for _, imp := range loader.resolveImportsAndModuleAugmentations(file) { + importsAndAugmentations, resolutionsInFile := loader.resolveImportsAndModuleAugmentations(file) + for _, imp := range importsAndAugmentations { t.addSubTask(imp, false) } t.file = file + t.metadata = loader.loadSourceFileMetaData(file.Path()) + t.resolutionsInFile = resolutionsInFile loader.startTasks(t.subTasks) }) } -func (p *fileLoader) loadSourceFileMetaData(path tspath.Path) { - _, ok := p.sourceFileMetaDatas.Load(path) - if ok { - return - } - +func (p *fileLoader) loadSourceFileMetaData(path tspath.Path) *ast.SourceFileMetaData { packageJsonType := p.resolver.GetPackageJsonTypeIfApplicable(string(path)) impliedNodeFormat := ast.GetImpliedNodeFormatForFile(string(path), packageJsonType) - metadata := &ast.SourceFileMetaData{ + return &ast.SourceFileMetaData{ PackageJsonType: packageJsonType, ImpliedNodeFormat: impliedNodeFormat, } - - p.sourceFileMetaDatas.Store(path, metadata) } func (p *fileLoader) parseSourceFile(fileName string) *ast.SourceFile { path := tspath.ToPath(fileName, p.host.GetCurrentDirectory(), p.host.FS().UseCaseSensitiveFileNames()) sourceFile := p.host.GetSourceFile(fileName, path, p.compilerOptions.GetEmitScriptTarget()) - p.loadSourceFileMetaData(path) return sourceFile } @@ -265,16 +269,14 @@ func (p *fileLoader) resolveTripleslashPathReference(moduleName string, containi return tspath.NormalizePath(referencedFileName) } -func (p *fileLoader) resolveImportsAndModuleAugmentations(file *ast.SourceFile) []string { - toParse := make([]string, 0, len(file.Imports)) +func (p *fileLoader) resolveImportsAndModuleAugmentations(file *ast.SourceFile) ([]string, module.ModeAwareCache[*module.ResolvedModule]) { if len(file.Imports) > 0 || len(file.ModuleAugmentations) > 0 { + toParse := make([]string, 0, len(file.Imports)) moduleNames := getModuleNames(file) resolutions := p.resolveModuleNames(moduleNames, file) resolutionsInFile := make(module.ModeAwareCache[*module.ResolvedModule], len(resolutions)) - p.resolvedModules.Store(file.Path(), resolutionsInFile) - for i, resolution := range resolutions { resolvedFileName := resolution.ResolvedFileName // TODO(ercornel): !!!: check if from node modules @@ -306,8 +308,10 @@ func (p *fileLoader) resolveImportsAndModuleAugmentations(file *ast.SourceFile) toParse = append(toParse, resolvedFileName) } } + + return toParse, resolutionsInFile } - return toParse + return nil, nil } func (p *fileLoader) resolveModuleNames(entries []*ast.Node, file *ast.SourceFile) []*module.ResolvedModule { From f193fa885a36fc7c56574bbb34e913f8cb23cac7 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Wed, 9 Apr 2025 11:00:18 -0700 Subject: [PATCH 09/10] optimize --- internal/compiler/fileloader.go | 36 +++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/internal/compiler/fileloader.go b/internal/compiler/fileloader.go index f5a247bed0..2181eada06 100644 --- a/internal/compiler/fileloader.go +++ b/internal/compiler/fileloader.go @@ -5,6 +5,7 @@ import ( "iter" "slices" "strings" + "sync/atomic" "github.com/microsoft/typescript-go/internal/ast" "github.com/microsoft/typescript-go/internal/collections" @@ -27,8 +28,8 @@ type fileLoader struct { tasksByFileName collections.SyncMap[string, *parseTask] rootTasks []*parseTask - // fileCount atomic.Int32 - // libFileCount atomic.Int32 + totalFileCount atomic.Int32 + libFileCount atomic.Int32 } type processedFiles struct { @@ -69,24 +70,32 @@ func processAllProgramFiles( loader.wg.RunAndWait() - size := loader.tasksByFileName.Size() - resolvedModules := make(map[tspath.Path]module.ModeAwareCache[*module.ResolvedModule], size) - sourceFileMetaDatas := make(map[tspath.Path]*ast.SourceFileMetaData, size) + totalFileCount := int(loader.totalFileCount.Load()) + libFileCount := int(loader.libFileCount.Load()) + + files := make([]*ast.SourceFile, 0, totalFileCount-libFileCount) + libFiles := make([]*ast.SourceFile, 0, totalFileCount) // totalFileCount here since we append files to it later to construct the final list + + resolvedModules := make(map[tspath.Path]module.ModeAwareCache[*module.ResolvedModule], totalFileCount) + sourceFileMetaDatas := make(map[tspath.Path]*ast.SourceFileMetaData, totalFileCount) - files, libFiles := []*ast.SourceFile{}, []*ast.SourceFile{} for task := range loader.collectTasks(loader.rootTasks) { + file := task.file if task.isLib { - libFiles = append(libFiles, task.file) + libFiles = append(libFiles, file) } else { - files = append(files, task.file) + files = append(files, file) } - resolvedModules[task.file.Path()] = task.resolutionsInFile - sourceFileMetaDatas[task.file.Path()] = task.metadata + path := file.Path() + resolvedModules[path] = task.resolutionsInFile + sourceFileMetaDatas[path] = task.metadata } loader.sortLibs(libFiles) + allFiles := append(libFiles, files...) + return processedFiles{ - files: append(libFiles, files...), + files: allFiles, resolvedModules: resolvedModules, sourceFileMetaDatas: sourceFileMetaDatas, } @@ -199,6 +208,11 @@ type parseTask struct { } func (t *parseTask) start(loader *fileLoader) { + loader.totalFileCount.Add(1) + if t.isLib { + loader.libFileCount.Add(1) + } + loader.wg.Queue(func() { file := loader.parseSourceFile(t.normalizedFilePath) From de64d249b5d575342cbd28e0e773ae10170bb40e Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Wed, 9 Apr 2025 11:05:09 -0700 Subject: [PATCH 10/10] optimize --- internal/compiler/fileloader.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/compiler/fileloader.go b/internal/compiler/fileloader.go index 2181eada06..b07bf50f41 100644 --- a/internal/compiler/fileloader.go +++ b/internal/compiler/fileloader.go @@ -215,6 +215,10 @@ func (t *parseTask) start(loader *fileLoader) { loader.wg.Queue(func() { file := loader.parseSourceFile(t.normalizedFilePath) + t.file = file + loader.wg.Queue(func() { + t.metadata = loader.loadSourceFileMetaData(file.Path()) + }) // !!! if noResolve, skip all of this t.subTasks = make([]*parseTask, 0, len(file.ReferencedFiles)+len(file.Imports)+len(file.ModuleAugmentations)) @@ -246,9 +250,8 @@ func (t *parseTask) start(loader *fileLoader) { t.addSubTask(imp, false) } - t.file = file - t.metadata = loader.loadSourceFileMetaData(file.Path()) t.resolutionsInFile = resolutionsInFile + loader.startTasks(t.subTasks) }) }