From 58609ca18ba0f962efbaa53d523650e18e8d5001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Louren=C3=A7o?= Date: Sat, 22 Oct 2022 22:54:08 -0300 Subject: [PATCH] fix(graceful-fs): added to solve issues with too many open files --- package-lock.json | 36 ++++++++++++++++++++++++++++-------- package.json | 2 ++ src/commands/run/index.ts | 23 ++++++++++++++--------- src/common/fs.ts | 13 +++++++++---- src/common/zip.ts | 13 +++++++++---- 5 files changed, 62 insertions(+), 25 deletions(-) diff --git a/package-lock.json b/package-lock.json index a22c3c3..aea8ef4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@h4ad/node-modules-packer", - "version": "1.2.0", + "version": "1.2.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@h4ad/node-modules-packer", - "version": "1.2.0", + "version": "1.2.1", "license": "MIT", "dependencies": { "@h4ad/dependency-extractor": "^1.0.0", @@ -17,6 +17,7 @@ "@oclif/plugin-plugins": "^2.1.0", "@oclif/plugin-version": "^1.1.1", "esbuild": "~0.15.7", + "graceful-fs": "^4.2.10", "rimraf": "^3.0.2", "semver": "^7.3.7", "yazl": "^2.5.1" @@ -31,6 +32,7 @@ "@semantic-release/github": "^8.0.4", "@types/bluebird": "^3.5.36", "@types/chai": "^4", + "@types/graceful-fs": "^4.1.5", "@types/mocha": "^9.0.0", "@types/mock-fs": "^4.13.1", "@types/node": "12.20.43", @@ -2091,6 +2093,15 @@ "@types/node": "*" } }, + "node_modules/@types/graceful-fs": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz", + "integrity": "sha512-anKkLmZZ+xm4p8JWBf4hElkM4XR+EZeA2M9BAkkTldmcyDY4mbdIJnRghDJH3Ov5ooY7/UAoENtmdMSkaAd7Cw==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz", @@ -7399,9 +7410,9 @@ } }, "node_modules/graceful-fs": { - "version": "4.2.9", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz", - "integrity": "sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ==" + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" }, "node_modules/grouped-queue": { "version": "2.0.0", @@ -19063,6 +19074,15 @@ "@types/node": "*" } }, + "@types/graceful-fs": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz", + "integrity": "sha512-anKkLmZZ+xm4p8JWBf4hElkM4XR+EZeA2M9BAkkTldmcyDY4mbdIJnRghDJH3Ov5ooY7/UAoENtmdMSkaAd7Cw==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/istanbul-lib-coverage": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz", @@ -22972,9 +22992,9 @@ } }, "graceful-fs": { - "version": "4.2.9", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz", - "integrity": "sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ==" + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" }, "grouped-queue": { "version": "2.0.0", diff --git a/package.json b/package.json index 72940f1..346007c 100644 --- a/package.json +++ b/package.json @@ -86,6 +86,7 @@ "@oclif/plugin-plugins": "^2.1.0", "@oclif/plugin-version": "^1.1.1", "esbuild": "~0.15.7", + "graceful-fs": "^4.2.10", "rimraf": "^3.0.2", "semver": "^7.3.7", "yazl": "^2.5.1" @@ -97,6 +98,7 @@ "@semantic-release/github": "^8.0.4", "@types/bluebird": "^3.5.36", "@types/chai": "^4", + "@types/graceful-fs": "^4.1.5", "@types/mocha": "^9.0.0", "@types/mock-fs": "^4.13.1", "@types/node": "12.20.43", diff --git a/src/commands/run/index.ts b/src/commands/run/index.ts index 80189e1..ba6b739 100644 --- a/src/commands/run/index.ts +++ b/src/commands/run/index.ts @@ -1,6 +1,5 @@ //#region Imports -import { existsSync, mkdirSync, readFileSync, statSync } from 'fs'; import { join, relative, resolve } from 'path'; import { DependencyInfo, @@ -15,6 +14,12 @@ import rimraf from 'rimraf'; import CustomCommand from '../../common/custom-command'; import CustomError from '../../common/custom-error'; import { defaultIgnoredFileExtensions } from '../../common/extensions'; +import { + safeExistsSync, + safeMkdirSync, + safeReadFileSync, + safeStatSync, +} from '../../common/fs'; import { HeadlessOptions } from '../../common/headless'; import { OutputInfo } from '../../common/output-info'; import { FasterZip, TransformAsyncCode, ZipArtifact } from '../../common/zip'; @@ -231,7 +236,7 @@ export default class Run extends CustomCommand { await this.zipDirectory(flags, dir, zipArtifacts, outputFilePath); - const size = statSync(outputFilePath).size; + const size = safeStatSync(outputFilePath).size; return { size, file: outputFile, path: outputPath }; } @@ -245,7 +250,7 @@ export default class Run extends CustomCommand { const nodeModulesFolderPath = join(dir, 'node_modules'); - if (existsSync(nodeModulesFolderPath)) { + if (safeExistsSync(nodeModulesFolderPath)) { this.logMessage(flags, 'log', 'Checking node folder... found'); return; } @@ -270,17 +275,17 @@ export default class Run extends CustomCommand { const resolvedOutputPath = resolve(dir, outputPath); const outputFilePath = join(resolvedOutputPath, outputFile); - if (!existsSync(resolvedOutputPath)) { + if (!safeExistsSync(resolvedOutputPath)) { this.logMessage( flags, 'log', `Not found folder in ${outputPath}, creating new one...`, ); - mkdirSync(resolvedOutputPath, { recursive: true }); + safeMkdirSync(resolvedOutputPath, { recursive: true }); } - if (!existsSync(outputFilePath)) { + if (!safeExistsSync(outputFilePath)) { this.logMessage( flags, 'debug', @@ -357,7 +362,7 @@ export default class Run extends CustomCommand { ): ExtractorContainer { const packageLockFilePath = join(dir, 'package-lock.json'); - if (existsSync(packageLockFilePath)) { + if (safeExistsSync(packageLockFilePath)) { this.logMessage( flags, 'debug', @@ -370,7 +375,7 @@ export default class Run extends CustomCommand { ); return new NpmExtractor().parse( - readFileSync(packageLockFilePath).toString('utf-8'), + safeReadFileSync(packageLockFilePath).toString('utf-8'), ); } @@ -499,7 +504,7 @@ export default class Run extends CustomCommand { const [relativePath, pathMappedTo] = includeFile.split(':'); const includeFilePath = join(dir, relativePath); - const stats = statSync(includeFilePath); + const stats = safeStatSync(includeFilePath); const metadataPath = pathMappedTo ? resolve('/', pathMappedTo).slice(1) : undefined; diff --git a/src/common/fs.ts b/src/common/fs.ts index 827d88e..dd194ad 100644 --- a/src/common/fs.ts +++ b/src/common/fs.ts @@ -1,6 +1,11 @@ -import fs from 'fs'; import { promisify } from 'util'; +import * as gracefulFs from 'graceful-fs'; -export const readFileAsync = promisify(fs.readFile); -export const readdirAsync = promisify(fs.readdir); -export const statAsync = promisify(fs.stat); +export const readdirAsync = promisify(gracefulFs.readdir); +export const statAsync = promisify(gracefulFs.stat); +export const safeCreateReadStream = gracefulFs.createReadStream; +export const safeCreateWriteStream = gracefulFs.createWriteStream; +export const safeExistsSync = gracefulFs.existsSync; +export const safeMkdirSync = gracefulFs.mkdirSync; +export const safeReadFileSync = gracefulFs.readFileSync; +export const safeStatSync = gracefulFs.statSync; diff --git a/src/common/zip.ts b/src/common/zip.ts index 86533d8..f34bab7 100644 --- a/src/common/zip.ts +++ b/src/common/zip.ts @@ -1,7 +1,12 @@ -import { Stats, createReadStream, createWriteStream } from 'fs'; +import { Stats } from 'fs'; import { join, normalize, relative } from 'path'; import { ZipFile } from 'yazl'; -import { readdirAsync, statAsync } from './fs'; +import { + readdirAsync, + safeCreateReadStream, + safeCreateWriteStream, + statAsync, +} from './fs'; import { StringStream } from './string-stream'; import { streamToUInt8Array } from './string-to-uint'; @@ -31,7 +36,7 @@ export class FasterZip { await new Promise((resolve, reject) => { (async () => { const zipfile = new ZipFile(); - const stream = createWriteStream(outputPath).once('error', reject); + const stream = safeCreateWriteStream(outputPath).once('error', reject); zipfile.outputStream.pipe(stream); @@ -104,7 +109,7 @@ export class FasterZip { const transformer = source.transformer && source.transformer(filePath, metadataPath); - const readStream = createReadStream(filePath).once('error', err => + const readStream = safeCreateReadStream(filePath).once('error', err => onErrorOnStream(err), );