Improved high memory usage cenario

.gitignore

@@ -2,3 +2,4 @@
 dist
 /package-lock.json
 *.swp
+/yarn.lock

README.md

@@ -252,12 +252,14 @@ const prerenderedHtml = require('!prerender-loader?string!./app.js');
 
 All options are ... optional.
 
-| Option        | Type    | Default            | Description                                                            |
-| ------------- | ------- | ------------------ | ---------------------------------------------------------------------- |
-| `string`      | boolean | false              | Output a JS module exporting an HTML String instead of the HTML itself |
-| `disabled`    | boolean | false              | Bypass the loader entirely (but still respect `options.string`)        |
-| `documentUrl` | string  | 'http://localhost' | Change the jsdom's URL (affects `window.location`, `document.URL`...)  |
-| `params`      | object  | null               | Options to pass to your prerender function                             |
+| Option             | Type    | Default            | Description                                                                         |
+| ------------------ | ------- | ------------------ | ----------------------------------------------------------------------------------- |
+| `string`           | boolean | false              | Output a JS module exporting an HTML String instead of the HTML itself              |
+| `disabled`         | boolean | false              | Bypass the loader entirely (but still respect `options.string`)                     |
+| `documentUrl`      | string  | 'http://localhost' | Change the jsdom's URL (affects `window.location`, `document.URL`...)               |
+| `params`           | object  | null               | Options to pass to your prerender function                                          |
+| `maxParallelTasks` | number  | null               | Limit the number o prerenders to execute in parallel (this should save some memory) |
+| `log`              | boolean | false              | Logs the start and end of the prerenders                                            |
 
 
 ---

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nettoolkit/prerender-loader",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "Painless universal prerendering for Webpack 5. Works great with html-webpack-plugin.",
   "main": "src/index.js",
   "source": "src/index.js",
@@ -32,7 +32,7 @@
   "homepage": "https://github.com/nettoolkit/prerender-loader#readme",
   "scripts": {
     "docs": "documentation readme -q --no-markdown-toc -a public -s Usage --sort-order alpha src",
-    "test": "jest --converage"
+    "test": "jest --coverage"
   },
   "babel": {
     "presets": [

src/index.js

@@ -51,6 +51,11 @@ const FILENAME = 'ssr-bundle.js';
 // Searches for fields of the form {{prerender}} or {{prerender:./some/module}}
 const PRERENDER_REG = /\{\{prerender(?::\s*([^}]+?)\s*)?\}\}/;
 
+/** 
+ * Controls how many prerender occur in parallel 
+ * @type {Promise[]}*/
+let parallelPromises;
+
 /**
  * prerender-loader can be applied to any HTML or JS file with the given options.
  * @public
@@ -73,13 +78,21 @@ const PRERENDER_REG = /\{\{prerender(?::\s*([^}]+?)\s*)?\}\}/;
  * import prerenderedHtml from '!prerender-loader!./file.html';
  */
 function PrerenderLoader (content) {
+  /** @type {{maxParallelTasks?: number, log?: boolean}} */
   const options = loaderUtils.getOptions(this) || {};
   const outputFilter = options.as === 'string' || options.string ? stringToModule : String;
 
   if (options.disabled === true) {
     return outputFilter(content);
   }
 
+  if (options.maxParallelTasks) {
+    if (!parallelPromises)
+      parallelPromises = new Array(options.maxParallelTasks)
+        .fill(undefined)
+        .map(() => Promise.resolve());
+  }
+
   // When applied to HTML, attempts to inject into a specified {{prerender}} field.
   // @note: this is only used when the entry module exports a String or function
   // that resolves to a String, otherwise the whole document is serialized.
@@ -95,14 +108,33 @@ function PrerenderLoader (content) {
 
   const callback = this.async();
 
-  prerender(this._compilation, this.request, options, inject, this)
-    .then(output => {
-      callback(null, outputFilter(output));
-    })
-    .catch(err => {
-      // console.error(err);
-      callback(err);
+  if (parallelPromises) {
+    const firstPromise = parallelPromises.shift()
+    const promiseToPush = firstPromise.then(() => {
+      if (options.log)
+        console.log("Starting prerender for entry", options.entry);
+      return prerender(this._compilation, this.request, options, inject, this)
+        .then((output) => {
+          callback(null, outputFilter(output));
+        })
+        .catch((err) => {
+          // console.error(err);
+          callback(err);
+        })
+        .finally(() => {
+          if (options.log) console.log("Finished prerender for", options.entry);
+        });
     });
+    parallelPromises.push(promiseToPush)
+  } else
+    prerender(this._compilation, this.request, options, inject, this)
+      .then((output) => {
+        callback(null, outputFilter(output));
+      })
+      .catch((err) => {
+        // console.error(err);
+        callback(err);
+      });
 }
 
 async function prerender (parentCompilation, request, options, inject, loader) {
@@ -114,7 +146,8 @@ async function prerender (parentCompilation, request, options, inject, loader) {
   const outputOptions = {
     // fix: some plugins ignore/bypass outputfilesystem, so use a temp directory and ignore any writes.
     path: os.tmpdir(),
-    filename: FILENAME
+    filename: FILENAME,
+    publicPath: parentCompiler.options.output.publicPath
   };
 
   // Only copy over allowed plugins (excluding them breaks extraction entirely).
@@ -145,24 +178,6 @@ async function prerender (parentCompilation, request, options, inject, loader) {
   // Kick off compilation at our entry module (either the parent compiler's entry or a custom one defined via `{{prerender:entry.js}}`)
   applyEntry(context, entry, compiler);
 
-  // NOTE: compilation.cache is deprecated in webpack 5.
-  // All tests appear to pass without setting up a subcache.
-  // What was the purpose of this subcache?
-  //
-  // Set up cache inheritance for the child compiler
-  // const subCache = 'subcache ' + request;
-  // function addChildCache (compilation, data) {
-  //   if (compilation.cache) {
-  //     if (!compilation.cache[subCache]) compilation.cache[subCache] = {};
-  //     compilation.cache = compilation.cache[subCache];
-  //   }
-  // }
-  // if (compiler.hooks) {
-  //   compiler.hooks.compilation.tap(PLUGIN_NAME, addChildCache);
-  // } else {
-  //   compiler.plugin('compilation', addChildCache);
-  // }
-
   const compilation = await runChildCompiler(compiler);
   let result;
   let dom, window, injectParent, injectNextSibling;

src/util.js

@@ -23,9 +23,16 @@ const path = require('path');
 function runChildCompiler (compiler) {
   return new Promise((resolve, reject) => {
     compiler.compile((err, compilation) => {
+      /** Only registering the compilation when there is error, because it consumes a lot of memory if there are multiple prerenders/entries ocurring during a compilation */
+      function addCompilationToParent() {
+        compiler.parentCompilation.children.push(compilation);
+      }
+
       // still allow the parent compiler to track execution of the child:
-      compiler.parentCompilation.children.push(compilation);
-      if (err) return reject(err);
+      if (err) {
+        addCompilationToParent()
+        return reject(err);
+      }
 
       // Bubble stat errors up and reject the Promise:
       if (compilation.errors && compilation.errors.length) {
@@ -41,6 +48,7 @@ function runChildCompiler (compiler) {
           }
           return error;
         }).join('\n');
+        addCompilationToParent()
         return reject(Error('Child compilation failed:\n' + errorDetails));
       }