Allow to write large files (> 256 MB)

src/typings/iconv-lite.d.ts

@@ -6,13 +6,13 @@
 /// <reference path='./node.d.ts'/>
 
 declare module 'iconv-lite' {
-	export function decode(buffer: NodeBuffer, encoding: string, options?: any): string;
+	export function decode(buffer: NodeBuffer, encoding: string): string;
 
-	export function encode(content: string, encoding: string, options?: any): NodeBuffer;
+	export function encode(content: string, encoding: string, options?: { addBOM?: boolean }): NodeBuffer;
 
 	export function encodingExists(encoding: string): boolean;
 
 	export function decodeStream(encoding: string): NodeJS.ReadWriteStream;
 
-	export function encodeStream(encoding: string): NodeJS.ReadWriteStream;
+	export function encodeStream(encoding: string, options?: { addBOM?: boolean }): NodeJS.ReadWriteStream;
 }

src/vs/base/node/encoding.ts

@@ -28,11 +28,11 @@ export function bomLength(encoding: string): number {
 	return 0;
 }
 
-export function decode(buffer: NodeBuffer, encoding: string, options?: any): string {
-	return iconv.decode(buffer, toNodeEncoding(encoding), options);
+export function decode(buffer: NodeBuffer, encoding: string): string {
+	return iconv.decode(buffer, toNodeEncoding(encoding));
 }
 
-export function encode(content: string, encoding: string, options?: any): NodeBuffer {
+export function encode(content: string, encoding: string, options?: { addBOM?: boolean }): NodeBuffer {
 	return iconv.encode(content, toNodeEncoding(encoding), options);
 }
 
@@ -44,6 +44,10 @@ export function decodeStream(encoding: string): NodeJS.ReadWriteStream {
 	return iconv.decodeStream(toNodeEncoding(encoding));
 }
 
+export function encodeStream(encoding: string, options?: { addBOM?: boolean }): NodeJS.ReadWriteStream {
+	return iconv.encodeStream(toNodeEncoding(encoding), options);
+}
+
 function toNodeEncoding(enc: string): string {
 	if (enc === UTF8_with_bom) {
 		return UTF8; // iconv does not distinguish UTF 8 with or without BOM, so we need to help it

src/vs/base/node/extfs.ts

@@ -54,7 +54,7 @@ export function copy(source: string, target: string, callback: (error: Error) =>
 		}
 
 		if (!stat.isDirectory()) {
-			return pipeFs(source, target, stat.mode & 511, callback);
+			return doCopyFile(source, target, stat.mode & 511, callback);
 		}
 
 		if (copiedSources[source]) {
@@ -75,6 +75,38 @@ export function copy(source: string, target: string, callback: (error: Error) =>
 	});
 }
 
+function doCopyFile(source: string, target: string, mode: number, callback: (error: Error) => void): void {
+	const reader = fs.createReadStream(source);
+	const writer = fs.createWriteStream(target, { mode });
+
+	let finished = false;
+	const finish = (error?: Error) => {
+		if (!finished) {
+			finished = true;
+
+			// in error cases, pass to callback
+			if (error) {
+				callback(error);
+			}
+
+			// we need to explicitly chmod because of https://github.com/nodejs/node/issues/1104
+			else {
+				fs.chmod(target, mode, callback);
+			}
+		}
+	};
+
+	// handle errors properly
+	reader.once('error', error => finish(error));
+	writer.once('error', error => finish(error));
+
+	// we are done (underlying fd has been closed)
+	writer.once('close', () => finish());
+
+	// start piping
+	reader.pipe(writer);
+}
+
 export function mkdirp(path: string, mode?: number): TPromise<boolean> {
 	const mkdir = () => nfcall(fs.mkdir, path, mode)
 		.then(null, (err: NodeJS.ErrnoException) => {
@@ -88,11 +120,12 @@ export function mkdirp(path: string, mode?: number): TPromise<boolean> {
 			return TPromise.wrapError<boolean>(err);
 		});
 
-	// is root?
+	// stop at root
 	if (path === paths.dirname(path)) {
 		return TPromise.as(true);
 	}
 
+	// recursively mkdir
 	return mkdir().then(null, (err: NodeJS.ErrnoException) => {
 		if (err.code === 'ENOENT') {
 			return mkdirp(paths.dirname(path), mode).then(mkdir);
@@ -102,40 +135,6 @@ export function mkdirp(path: string, mode?: number): TPromise<boolean> {
 	});
 }
 
-function pipeFs(source: string, target: string, mode: number, callback: (error: Error) => void): void {
-	let callbackHandled = false;
-
-	const readStream = fs.createReadStream(source);
-	const writeStream = fs.createWriteStream(target, { mode: mode });
-
-	const onError = (error: Error) => {
-		if (!callbackHandled) {
-			callbackHandled = true;
-			callback(error);
-		}
-	};
-
-	readStream.on('error', onError);
-	writeStream.on('error', onError);
-
-	readStream.on('end', () => {
-		(<any>writeStream).end(() => { // In this case the write stream is known to have an end signature with callback
-			if (!callbackHandled) {
-				callbackHandled = true;
-
-				fs.chmod(target, mode, callback); // we need to explicitly chmod because of https://github.com/nodejs/node/issues/1104
-			}
-		});
-	});
-
-	// In node 0.8 there is no easy way to find out when the pipe operation has finished. As such, we use the end property = false
-	// so that we are in charge of calling end() on the write stream and we will be notified when the write stream is really done.
-	// We can do this because file streams have an end() method that allows to pass in a callback.
-	// In node 0.10 there is an event 'finish' emitted from the write stream that can be used. See
-	// https://groups.google.com/forum/?fromgroups=#!topic/nodejs/YWQ1sRoXOdI
-	readStream.pipe(writeStream, { end: false });
-}
-
 // Deletes the given path by first moving it out of the workspace. This has two benefits. For one, the operation can return fast because
 // after the rename, the contents are out of the workspace although not yet deleted. The greater benefit however is that this operation
 // will fail in case any file is used by another process. fs.unlink() in node will not bail if a file unlinked is used by another process.
@@ -320,15 +319,101 @@ export function mv(source: string, target: string, callback: (error: Error) => v
 	});
 }
 
+let canFlush = true;
+export function writeFileAndFlush(path: string, data: string | NodeBuffer | NodeJS.ReadableStream, options: { mode?: number; flag?: string; }, callback: (error?: Error) => void): void {
+	options = ensureOptions(options);
+
+	if (typeof data === 'string' || Buffer.isBuffer(data)) {
+		doWriteFileAndFlush(path, data, options, callback);
+	} else {
+		doWriteFileStreamAndFlush(path, data, options, callback);
+	}
+}
+
+function doWriteFileStreamAndFlush(path: string, reader: NodeJS.ReadableStream, options: { mode?: number; flag?: string; }, callback: (error?: Error) => void): void {
+
+	// finish only once
+	let finished = false;
+	const finish = (error?: Error) => {
+		if (!finished) {
+			finished = true;
+
+			// in error cases we need to manually close streams
+			// if the write stream was successfully opened
+			if (error) {
+				if (isOpen) {
+					writer.once('close', () => callback(error));
+					writer.close();
+				} else {
+					callback(error);
+				}
+			}
+
+			// otherwise just return without error
+			else {
+				callback();
+			}
+		}
+	};
+
+	// create writer to target. we set autoClose: false because we want to use the streams
+	// file descriptor to call fs.fdatasync to ensure the data is flushed to disk
+	const writer = fs.createWriteStream(path, { mode: options.mode, flags: options.flag, autoClose: false });
+
+	// Event: 'open'
+	// Purpose: save the fd for later use
+	// Notes: will not be called when there is an error opening the file descriptor!
+	let fd: number;
+	let isOpen: boolean;
+	writer.once('open', descriptor => {
+		fd = descriptor;
+		isOpen = true;
+	});
+
+	// Event: 'error'
+	// Purpose: to return the error to the outside and to close the write stream (does not happen automatically)
+	reader.once('error', error => finish(error));
+	writer.once('error', error => finish(error));
+
+	// Event: 'finish'
+	// Purpose: use fs.fdatasync to flush the contents to disk
+	// Notes: event is called when the writer has finished writing to the underlying resource. we must call writer.close()
+	// because we have created the WriteStream with autoClose: false
+	writer.once('finish', () => {
+
+		// flush to disk
+		if (canFlush && isOpen) {
+			fs.fdatasync(fd, (syncError: Error) => {
+
+				// In some exotic setups it is well possible that node fails to sync
+				// In that case we disable flushing and warn to the console
+				if (syncError) {
+					console.warn('[node.js fs] fdatasync is now disabled for this session because it failed: ', syncError);
+					canFlush = false;
+				}
+
+				writer.close();
+			});
+		} else {
+			writer.close();
+		}
+	});
+
+	// Event: 'close'
+	// Purpose: signal we are done to the outside
+	// Notes: event is called when the writer's filedescriptor is closed
+	writer.once('close', () => finish());
+
+	// start data piping
+	reader.pipe(writer);
+}
+
 // Calls fs.writeFile() followed by a fs.sync() call to flush the changes to disk
 // We do this in cases where we want to make sure the data is really on disk and
 // not in some cache.
 //
 // See https://github.com/nodejs/node/blob/v5.10.0/lib/fs.js#L1194
-let canFlush = true;
-export function writeFileAndFlush(path: string, data: string | NodeBuffer, options: { mode?: number; flag?: string; }, callback: (error: Error) => void): void {
-	options = ensureOptions(options);
-
+function doWriteFileAndFlush(path: string, data: string | NodeBuffer, options: { mode?: number; flag?: string; }, callback: (error?: Error) => void): void {
 	if (!canFlush) {
 		return fs.writeFile(path, data, options, callback);
 	}

src/vs/base/node/pfs.ts

@@ -101,6 +101,7 @@ const writeFilePathQueue: { [path: string]: Queue<void> } = Object.create(null);
 
 export function writeFile(path: string, data: string, options?: { mode?: number; flag?: string; }): TPromise<void>;
 export function writeFile(path: string, data: NodeBuffer, options?: { mode?: number; flag?: string; }): TPromise<void>;
+export function writeFile(path: string, data: NodeJS.ReadableStream, options?: { mode?: number; flag?: string; }): TPromise<void>;
 export function writeFile(path: string, data: any, options?: { mode?: number; flag?: string; }): TPromise<void> {
 	let queueKey = toQueueKey(path);