perf(cli): reduce memory consumption

This change is intended to prevent "maxBuffer exceeded" panics by: + Processing files in batches of 300. + Filtering files using a negation pattern in globby, instead of retrieving all files then filtering the returned array. Closes #173
JamieMason · Sep 15, 2019 · 3166245 · 3166245
1 parent 6c3d799
commit 3166245
Show file tree

Hide file tree

Showing 5 changed files with 66 additions and 31 deletions.
diff --git a/src/get-stats.ts b/src/get-stats.ts
@@ -49,7 +49,7 @@ const createStat = (label: string, sizeAfter: number, sizeBefore: number) => {
 
 export const getStats = async (options: IOptions): Promise<IStats> => {
   const fileStats: IFileStats[] = await Promise.all(
-    options.files.supported.map(async ({ source, tmp }) => {
+    options.filePaths.map(async ({ source, tmp }) => {
       const sizeBefore = await getFileSize(source);
       const sizeAfter = await getFileSize(tmp);
       return createStat(source, sizeAfter, sizeBefore);

diff --git a/src/imageoptim.ts b/src/imageoptim.ts
@@ -82,13 +82,14 @@ if (process.platform !== 'darwin') {
   console.log('imageoptim-cli is macOS only');
 }
 
+const supportedTypesPattern = SUPPORTED_FILE_TYPES.map((fileType) => `*${fileType}`).join('|');
+
+patterns.push(`!**/!(${supportedTypesPattern})`);
+
 const filePaths = sync(patterns.map((pattern) => pattern.replace('~', homedir())));
-const supportedFilePaths = filePaths.filter(isSupported(SUPPORTED_FILE_TYPES)).map((filePath) => ({
-  source: filePath,
-  tmp: join(TMPDIR, filePath)
-}));
 
 cli({
+  batchSize: 300,
   enabled: {
     color: program.color === true,
     imageAlpha: program.imagealpha === true,
@@ -97,10 +98,7 @@ cli({
     quit: program.quit === true,
     stats: program.stats === true
   },
-  files: {
-    all: filePaths,
-    supported: supportedFilePaths
-  },
+  filePaths,
   numberOfColors: program.numberOfColors || PNGQUANT_NUMBER_OF_COLORS,
   quality: program.quality || PNGQUANT_QUALITY,
   speed: program.speed || PNGQUANT_SPEED,

diff --git a/src/index.ts b/src/index.ts
@@ -1,3 +1,4 @@
+import { join } from 'path';
 import { getStats } from './get-stats';
 import { bug, complete, enableColor, warning } from './log';
 import { runImageAlpha } from './run-imagealpha';
@@ -13,7 +14,8 @@ export interface IFile {
   tmp: string;
 }
 
-export interface IOptions {
+export interface ICliOptions {
+  batchSize: number;
   enabled: {
     color: boolean;
     imageAlpha: boolean;
@@ -22,10 +24,24 @@ export interface IOptions {
     quit: boolean;
     stats: boolean;
   };
-  files: {
-    all: string[];
-    supported: IFile[];
+  filePaths: string[];
+  numberOfColors: string;
+  quality: string;
+  speed: string;
+  tmpDir: string;
+}
+
+export interface IOptions {
+  batchSize: number;
+  enabled: {
+    color: boolean;
+    imageAlpha: boolean;
+    imageOptim: boolean;
+    jpegMini: boolean;
+    quit: boolean;
+    stats: boolean;
   };
+  filePaths: IFile[];
   numberOfColors: string;
   quality: string;
   speed: string;
@@ -39,22 +55,43 @@ const runnersByName = {
   stats: getStats
 };
 
-export const cli = async (options: IOptions) => {
-  try {
-    const runIfEnabled = (key: keyof typeof runnersByName) =>
-      options.enabled[key] ? runnersByName[key](options) : Promise.resolve();
+const cloneArray = (array: string[]) => [...array];
 
+const runIfEnabled = (key: keyof typeof runnersByName, options: IOptions) =>
+  options.enabled[key] ? runnersByName[key](options) : Promise.resolve();
+
+const processBatch = async (options: IOptions) => {
+  await setup(options);
+  await Promise.all([runIfEnabled('imageAlpha', options), runIfEnabled('jpegMini', options)]);
+  await runIfEnabled('imageOptim', options);
+  const stats = await runIfEnabled('stats', options);
+  await tearDown(options);
+  if (stats) {
+    await writeReport(stats);
+  }
+};
+
+export const cli = async (options: ICliOptions) => {
+  try {
+    const filesMutable = cloneArray(options.filePaths);
     enableColor(options.enabled.color);
-    if (options.files.supported.length === 0) {
+    if (filesMutable.length === 0) {
       return warning('No images matched the patterns provided');
     }
-    await setup(options);
-    await Promise.all([runIfEnabled('imageAlpha'), runIfEnabled('jpegMini')]);
-    await runIfEnabled('imageOptim');
-    const stats = await runIfEnabled('stats');
-    await tearDown(options);
-    if (stats) {
-      await writeReport(stats);
+    while (filesMutable.length > 0) {
+      const filePaths = filesMutable.splice(0, options.batchSize);
+      await processBatch({
+        batchSize: options.batchSize,
+        enabled: options.enabled,
+        filePaths: filePaths.map((filePath) => ({
+          source: filePath,
+          tmp: join(options.tmpDir, filePath)
+        })),
+        numberOfColors: options.numberOfColors,
+        quality: options.quality,
+        speed: options.speed,
+        tmpDir: options.tmpDir
+      });
     }
     complete('Finished');
   } catch (err) {

diff --git a/src/run-imagealpha.ts b/src/run-imagealpha.ts
@@ -7,7 +7,7 @@ import { pngquant } from './pngquant';
 
 export const runImageAlpha: AppRunner = async (options: IOptions) => {
   info(`Running ${IMAGEALPHA.name}...`);
-  const pngFilePaths = options.files.supported
+  const pngFilePaths = options.filePaths
     .map((file) => file.tmp)
     .filter(isSupported(IMAGEALPHA.supports));
   if (!(await pathExists(PNGQUANT_BIN_PATH))) {

diff --git a/src/tmpdir.ts b/src/tmpdir.ts
@@ -5,18 +5,18 @@ import { verbose } from './log';
 const sourceToTmp = ({ source, tmp }: IFile) => copy(source, tmp);
 const tmpToSource = ({ source, tmp }: IFile) => copy(tmp, source);
 
-export const clean = (options: IOptions) => remove(options.tmpDir);
+export const clean = (options: { tmpDir: string }) => remove(options.tmpDir);
 
 export const setup = async (options: IOptions) => {
   verbose(`Creating temp directory ${options.tmpDir}`);
   await clean(options);
-  verbose(`Copying ${options.files.supported.length} files to temp directory`);
-  await Promise.all(options.files.supported.map(sourceToTmp));
+  verbose(`Copying ${options.filePaths.length} files to temp directory`);
+  await Promise.all(options.filePaths.map(sourceToTmp));
 };
 
 export const tearDown = async (options: IOptions) => {
-  verbose(`Copying ${options.files.supported.length} files back to original location`);
-  await Promise.all(options.files.supported.map(tmpToSource));
+  verbose(`Copying ${options.filePaths.length} files back to original location`);
+  await Promise.all(options.filePaths.map(tmpToSource));
   verbose(`Deleting temp directory ${options.tmpDir}`);
   await clean(options);
 };