From 9882ae373cdc96416a3724fe93d0df69333605e2 Mon Sep 17 00:00:00 2001 From: Evan Wallace Date: Tue, 25 Aug 2020 20:44:30 -0700 Subject: [PATCH] prevent ulimit overflow when writing too (#348) --- cmd/esbuild/service.go | 4 ++++ internal/fs/fs.go | 20 ++++++++++---------- pkg/api/api_impl.go | 2 ++ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/cmd/esbuild/service.go b/cmd/esbuild/service.go index 921fcdc3a10..9df8bbafafe 100644 --- a/cmd/esbuild/service.go +++ b/cmd/esbuild/service.go @@ -255,18 +255,22 @@ func (service *serviceType) handleTransformRequest(id uint32, request map[string if inputFS && len(result.JS) > 0 { file := input + ".js" + fs.BeforeFileOpen() if err := ioutil.WriteFile(file, result.JS, 0644); err == nil { result.JS = []byte(file) jsFS = true } + fs.AfterFileClose() } if inputFS && len(result.JSSourceMap) > 0 { file := input + ".map" + fs.BeforeFileOpen() if err := ioutil.WriteFile(file, result.JSSourceMap, 0644); err == nil { result.JSSourceMap = []byte(file) jsSourceMapFS = true } + fs.AfterFileClose() } return encodePacket(packet{ diff --git a/internal/fs/fs.go b/internal/fs/fs.go index 644b6fbf0fe..42c261dedd1 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -48,8 +48,8 @@ func (e *Entry) stat() { entryPath := filepath.Join(e.dir, e.base) // Use "lstat" since we want information about symbolic links - beforeFileOpen() - defer afterFileClose() + BeforeFileOpen() + defer AfterFileClose() stat, err := os.Lstat(entryPath) if err != nil { return @@ -232,12 +232,12 @@ type realFS struct { // Limit the number of files open simultaneously to avoid ulimit issues var fileOpenLimit = make(chan bool, 32) -func beforeFileOpen() { +func BeforeFileOpen() { // This will block if the number of open files is already at the limit fileOpenLimit <- false } -func afterFileClose() { +func AfterFileClose() { <-fileOpenLimit } @@ -248,8 +248,8 @@ func realpath(path string) string { } dir = realpath(dir) path = filepath.Join(dir, filepath.Base(path)) - beforeFileOpen() - defer afterFileClose() + BeforeFileOpen() + defer AfterFileClose() if link, err := os.Readlink(path); err == nil { if filepath.IsAbs(link) { return link @@ -316,8 +316,8 @@ func (fs *realFS) ReadDirectory(dir string) map[string]*Entry { } func (fs *realFS) ReadFile(path string) (string, bool) { - beforeFileOpen() - defer afterFileClose() + BeforeFileOpen() + defer AfterFileClose() buffer, err := ioutil.ReadFile(path) return string(buffer), err == nil } @@ -355,8 +355,8 @@ func (*realFS) Rel(base string, target string) (string, bool) { } func readdir(dirname string) ([]string, error) { - beforeFileOpen() - defer afterFileClose() + BeforeFileOpen() + defer AfterFileClose() f, err := os.Open(dirname) if err != nil { return nil, err diff --git a/pkg/api/api_impl.go b/pkg/api/api_impl.go index 9e9d90cf4ea..dbc74956577 100644 --- a/pkg/api/api_impl.go +++ b/pkg/api/api_impl.go @@ -546,6 +546,8 @@ func buildImpl(buildOpts BuildOptions) BuildResult { waitGroup.Add(len(outputFiles)) for _, outputFile := range outputFiles { go func(outputFile OutputFile) { + fs.BeforeFileOpen() + defer fs.AfterFileClose() if err := os.MkdirAll(filepath.Dir(outputFile.Path), 0755); err != nil { log.AddError(nil, ast.Loc{}, fmt.Sprintf( "Failed to create output directory: %s", err.Error()))