Add support for streaming output with option (#118)

Co-authored-by: Sindre Sorhus <sindresorhus@gmail.com>

index.js

@@ -1,5 +1,6 @@
 'use strict';
 const path = require('path');
+const BufferConstants = require('buffer').constants;
 const Vinyl = require('vinyl');
 const PluginError = require('plugin-error');
 const through = require('through2');
@@ -13,6 +14,7 @@ module.exports = (filename, options) => {
 
 	options = {
 		compress: true,
+		buffer: true,
 		...options
 	};
 
@@ -64,7 +66,22 @@ module.exports = (filename, options) => {
 		}
 
 		(async () => {
-			const data = await getStream.buffer(zip.outputStream);
+			let data;
+			if (options.buffer) {
+				try {
+					data = await getStream.buffer(zip.outputStream, {maxBuffer: BufferConstants.MAX_LENGTH});
+				} catch (error) {
+					if (error instanceof getStream.MaxBufferError) {
+						callback(new PluginError('gulp-zip', 'The output ZIP file is too big to store in a buffer (larger than Buffer MAX_LENGTH). To output a stream instead, set the gulp-zip buffer option to `false`.'));
+					} else {
+						callback(error);
+					}
+
+					return;
+				}
+			} else {
+				data = zip.outputStream;
+			}
 
 			this.push(new Vinyl({
 				cwd: firstFile.cwd,

package.json

@@ -28,7 +28,7 @@
 		"file"
 	],
 	"dependencies": {
-		"get-stream": "^5.1.0",
+		"get-stream": "^5.2.0",
 		"plugin-error": "^1.0.1",
 		"through2": "^3.0.1",
 		"vinyl": "^2.1.0",

readme.md

@@ -51,3 +51,13 @@ Default: `undefined`
 Overrides the modification timestamp for all files added to the archive.
 
 Tip: Setting it to the same value across executions enables you to create stable archives that change only when the contents of their entries change, regardless of whether those entries were "touched" or regenerated.
+
+##### buffer
+
+Type: `boolean`<br>
+Default: `true`
+
+If `true`, the resulting ZIP file contents will be a buffer. Large zip files may not be possible to buffer, depending on the size of [Buffer MAX_LENGTH](https://nodejs.org/api/buffer.html#buffer_buffer_constants_max_length).
+If `false`, the ZIP file contents will be a stream.
+
+We use this option instead of relying on [gulp.src's `buffer` option](https://gulpjs.com/docs/en/api/src/#options) because we are mapping many input files to one output file and can't reliably detect what the output mode should be based on the inputs, since Vinyl streams could contain mixed streaming and buffered content.

test.js

@@ -1,5 +1,6 @@
 import fs from 'fs';
 import path from 'path';
+import {constants as BufferConstants} from 'buffer';
 import test from 'ava';
 import Vinyl from 'vinyl';
 import through2 from 'through2';
@@ -32,6 +33,7 @@ test.cb('should zip files', t => {
 	stream.on('data', file => {
 		t.is(path.normalize(file.path), path.join(__dirname, 'fixture/test.zip'));
 		t.is(file.relative, 'test.zip');
+		t.true(file.isBuffer());
 		t.true(file.contents.length > 0);
 	});
 
@@ -225,3 +227,72 @@ test.cb('when `options.modifiedTime` is specified, should create identical zips
 		}
 	}));
 });
+
+test.cb('should produce a buffer by default', t => {
+	const stream = zip('test.zip');
+	const file = vinylFile.readSync(path.join(__dirname, 'fixture/fixture.txt'));
+
+	stream.on('data', file => {
+		t.true(file.isBuffer());
+	});
+
+	stream.on('end', () => {
+		t.end();
+	});
+
+	stream.write(file);
+	stream.end();
+});
+
+test.cb('should produce a stream if requested', t => {
+	const stream = zip('test.zip', {buffer: false});
+	const file = vinylFile.readSync(path.join(__dirname, 'fixture/fixture.txt'));
+
+	stream.on('data', file => {
+		t.true(file.isStream());
+	});
+
+	stream.on('end', () => {
+		t.end();
+	});
+
+	stream.write(file);
+	stream.end();
+});
+
+test.cb('should explain buffer size errors', t => {
+	const stream = zip('test.zip', {compress: false});
+	const unzipper = unzip();
+	const stats = fs.statSync(path.join(__dirname, 'fixture/fixture.txt'));
+	stream.pipe(vinylAssign({extract: true})).pipe(unzipper);
+
+	stream.on('error', error => {
+		t.is(error.message, 'The output ZIP file is too big to store in a buffer (larger than Buffer MAX_LENGTH). To output a stream instead, set the gulp-zip buffer option to `false`.');
+		t.end();
+	});
+
+	function addFile(contents) {
+		stream.write(new Vinyl({
+			cwd: __dirname,
+			base: path.join(__dirname, 'fixture'),
+			path: path.join(__dirname, 'fixture/file.txt'),
+			contents,
+			stat: stats
+		}));
+	}
+
+	// Yazl internally enforces a lower max buffer size than MAX_LENGTH
+	const maxYazlBuffer = 1073741823;
+
+	// Produce some giant data files to exceed max length but staying under Yazl's maximum
+	const filesNeeded = Math.floor(BufferConstants.MAX_LENGTH / maxYazlBuffer);
+	for (let files = 0; files < filesNeeded; files++) {
+		addFile(Buffer.allocUnsafe(maxYazlBuffer));
+	}
+
+	// Pad all the way up to BufferConstants.MAX_LENGTH
+	addFile(Buffer.allocUnsafe(BufferConstants.MAX_LENGTH % maxYazlBuffer));
+
+	stream.end();
+});
+