1842: New xlsx option for ignoring certain nodes for improved perform…

…ance (#2132)

* 1842: New xlsx option for ignoring certain nodes for improved performance

* Included test file

* Added typescript typings

* optimizing ifs

* new line EOF

* Update README.md - available options

---------

Co-authored-by: Willem Le Roux <willemleroux@safeguardglobal.com>
Co-authored-by: Siemienik Pawel <hi@siemienik.com>

README.md

@@ -2159,6 +2159,12 @@ faster or more resilient.
 
 #### Reading XLSX[⬆](#contents)<!-- Link generated with jump2header -->
 
+Options supported when reading CSV files.
+
+| Field            |  Required   |    Type     |Description  |
+| ---------------- | ----------- | ----------- | ----------- |
+| ignoreNodes      |     N       |  Array      | A list of node names to ignore while loading the XLSX document. Improves performance in some situations. <br/> Available: `sheetPr`, `dimension`, `sheetViews `, `sheetFormatPr`, `cols `, `sheetData`, `autoFilter `, `mergeCells `, `rowBreaks`, `hyperlinks `, `pageMargins`, `dataValidations`, `pageSetup`, `headerFooter `, `printOptions `, `picture`, `drawing`, `sheetProtection`, `tableParts `, `conditionalFormatting`, `extLst`,|
+
 ```javascript
 // read from a file
 const workbook = new Excel.Workbook();
@@ -2176,6 +2182,16 @@ await workbook.xlsx.read(stream);
 const workbook = new Excel.Workbook();
 await workbook.xlsx.load(data);
 // ... use workbook
+
+
+// using additional options
+const workbook = new Excel.Workbook();
+await workbook.xlsx.load(data, {
+  ignoreNodes: [
+    'dataValidations' // ignores the workbook's Data Validations
+  ],
+});
+// ... use workbook
 ```
 
 #### Writing XLSX[⬆](#contents)<!-- Link generated with jump2header -->

index.d.ts

@@ -1457,6 +1457,13 @@ export interface JSZipGeneratorOptions {
 	};
 }
 
+export interface XlsxReadOptions {
+	/**
+	 * The list of XML node names to ignore while parsing an XLSX file
+	 */
+	ignoreNodes: string[];
+}
+
 export interface XlsxWriteOptions extends stream.xlsx.WorkbookWriterOptions {
 	/**
 	 * The option passed to JsZip#generateAsync(options)
@@ -1468,19 +1475,19 @@ export interface Xlsx {
 	/**
 	 * read from a file
 	 */
-	readFile(path: string): Promise<Workbook>;
+	readFile(path: string, options?: Partial<XlsxReadOptions>): Promise<Workbook>;
 
 	/**
 	 * read from a stream
 	 * @param stream
 	 */
-	read(stream: import('stream').Stream): Promise<Workbook>;
+	read(stream: import('stream').Stream, options?: Partial<XlsxReadOptions>): Promise<Workbook>;
 
 	/**
 	 * load from an array buffer
 	 * @param buffer
 	 */
-	load(buffer: Buffer): Promise<Workbook>;
+	load(buffer: Buffer, options?: Partial<XlsxReadOptions>): Promise<Workbook>;
 
 	/**
 	 * write to a buffer

lib/xlsx/xform/sheet/worksheet-xform.js

@@ -92,7 +92,10 @@ class WorkSheetXform extends BaseXform {
   constructor(options) {
     super();
 
-    const {maxRows, maxCols} = options || {};
+    const {maxRows, maxCols, ignoreNodes} = options || {};
+
+    this.ignoreNodes = ignoreNodes || [];
+
     this.map = {
       sheetPr: new SheetPropertiesXform(),
       dimension: new DimensionXform(),
@@ -221,9 +224,7 @@ class WorkSheetXform extends BaseXform {
           });
         }
         let rIdImage =
-          this.preImageId === medium.imageId
-            ? drawingRelsHash[medium.imageId]
-            : drawingRelsHash[drawing.rels.length];
+          this.preImageId === medium.imageId ? drawingRelsHash[medium.imageId] : drawingRelsHash[drawing.rels.length];
         if (!rIdImage) {
           rIdImage = nextRid(drawing.rels);
           drawingRelsHash[drawing.rels.length] = rIdImage;
@@ -368,8 +369,8 @@ class WorkSheetXform extends BaseXform {
       return true;
     }
 
-    this.parser = this.map[node.name];
-    if (this.parser) {
+    if (this.map[node.name] && !this.ignoreNodes.includes(node.name)) {
+      this.parser = this.map[node.name];
       this.parser.parseOpen(node);
     }
     return true;
@@ -405,11 +406,7 @@ class WorkSheetXform extends BaseXform {
             false,
           margins: this.map.pageMargins.model,
         };
-        const pageSetup = Object.assign(
-          sheetProperties,
-          this.map.pageSetup.model,
-          this.map.printOptions.model
-        );
+        const pageSetup = Object.assign(sheetProperties, this.map.pageSetup.model, this.map.printOptions.model);
         const conditionalFormattings = mergeConditionalFormattings(
           this.map.conditionalFormatting.model,
           this.map.extLst.model && this.map.extLst.model['x14:conditionalFormattings']

lib/xlsx/xlsx.js

@@ -22,7 +22,7 @@ const TableXform = require('./xform/table/table-xform');
 const CommentsXform = require('./xform/comment/comments-xform');
 const VmlNotesXform = require('./xform/comment/vml-notes-xform');
 
-const theme1Xml = require('./xml/theme1.js');
+const theme1Xml = require('./xml/theme1');
 
 function fsReadFileAsync(filename, options) {
   return new Promise((resolve, reject) => {
@@ -285,9 +285,11 @@ class XLSX {
           entryName = entryName.substr(1);
         }
         let stream;
-        if (entryName.match(/xl\/media\//) ||
+        if (
+          entryName.match(/xl\/media\//) ||
           // themes are not parsed as stream
-          entryName.match(/xl\/theme\/([a-zA-Z0-9]+)[.]xml/)) {
+          entryName.match(/xl\/theme\/([a-zA-Z0-9]+)[.]xml/)
+        ) {
           stream = new PassThrough();
           stream.write(await entry.async('nodebuffer'));
         } else {
@@ -597,8 +599,7 @@ class XLSX {
     model.created = model.created || new Date();
     model.modified = model.modified || new Date();
 
-    model.useSharedStrings =
-      options.useSharedStrings !== undefined ? options.useSharedStrings : true;
+    model.useSharedStrings = options.useSharedStrings !== undefined ? options.useSharedStrings : true;
     model.useStyles = options.useStyles !== undefined ? options.useStyles : true;
 
     // Manage the shared strings

spec/integration/data/.gitignore

@@ -0,0 +1 @@
+!*.xlsx

spec/integration/issues/issue-1842-dataValidations-memory-overload.spec.js

@@ -0,0 +1,32 @@
+const {join} = require('path');
+const {readFileSync} = require('fs');
+
+const ExcelJS = verquire('exceljs');
+
+const fileName = './spec/integration/data/test-issue-1842.xlsx';
+
+describe('github issues', () => {
+  describe('issue 1842 - Memory overload when unnecessary dataValidations apply', () => {
+    it('when using readFile', async () => {
+      const wb = new ExcelJS.Workbook();
+      await wb.xlsx.readFile(fileName, {
+        ignoreNodes: ['dataValidations'],
+      });
+
+      // arriving here is success
+      expect(true).to.equal(true);
+    });
+
+    it('when loading an in memory buffer', async () => {
+      const filePath = join(process.cwd(), fileName);
+      const buffer = readFileSync(filePath);
+      const wb = new ExcelJS.Workbook();
+      await wb.xlsx.load(buffer, {
+        ignoreNodes: ['dataValidations'],
+      });
+
+      // arriving here is success
+      expect(true).to.equal(true);
+    });
+  });
+});