Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fs: improve readFile performance #718

Closed
wants to merge 1 commit into from
Closed

fs: improve readFile performance #718

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
212 changes: 137 additions & 75 deletions lib/fs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

'use strict';

const SlowBuffer = require('buffer').SlowBuffer;
const util = require('util');
const pathModule = require('path');

Expand Down Expand Up @@ -218,101 +219,162 @@ fs.existsSync = function(path) {
fs.readFile = function(path, options, callback_) {
var callback = maybeCallback(arguments[arguments.length - 1]);

if (!options || typeof options === 'function') {
if (!options || typeof options === 'function')
options = { encoding: null, flag: 'r' };
} else if (typeof options === 'string') {
else if (typeof options === 'string')
options = { encoding: options, flag: 'r' };
} else if (typeof options !== 'object') {
else if (typeof options !== 'object')
throw new TypeError('Bad arguments');
}

var encoding = options.encoding;
assertEncoding(encoding);

// first, stat the file, so we know the size.
var size;
var buffer; // single buffer with file data
var buffers; // list for when size is unknown
var pos = 0;
var fd;

var flag = options.flag || 'r';
fs.open(path, flag, 0o666, function(er, fd_) {
if (er) return callback(er);
fd = fd_;

fs.fstat(fd, function(er, st) {
if (er) {
return fs.close(fd, function() {
callback(er);
});
}
if (!nullCheck(path, callback))
return;

size = st.size;
if (size === 0) {
// the kernel lies about many files.
// Go ahead and try to read some bytes.
buffers = [];
return read();
}
var context = new ReadFileContext(callback, encoding);
var req = new FSReqWrap();
req.context = context;
req.oncomplete = readFileAfterOpen;

if (size > kMaxLength) {
var err = new RangeError('File size is greater than possible Buffer: ' +
'0x3FFFFFFF bytes');
return fs.close(fd, function() {
callback(err);
});
}
buffer = new Buffer(size);
read();
});
});
binding.open(pathModule._makeLong(path),
stringToFlags(flag),
0o666,
req);
};

function read() {
if (size === 0) {
buffer = new Buffer(8192);
fs.read(fd, buffer, 0, 8192, -1, afterRead);
} else {
fs.read(fd, buffer, pos, size - pos, -1, afterRead);
}
const kReadFileBufferLength = 8 * 1024;

function ReadFileContext(callback, encoding) {
this.fd = undefined;
this.size = undefined;
this.callback = callback;
this.buffers = null;
this.buffer = null;
this.pos = 0;
this.encoding = encoding;
this.err = null;
}

ReadFileContext.prototype.read = function() {
var fd = this.fd;
var size = this.size;
var buffer;
var offset;
var length;

if (size === 0) {
buffer = this.buffer = new SlowBuffer(kReadFileBufferLength);
offset = 0;
length = kReadFileBufferLength;
} else {
buffer = this.buffer;
offset = this.pos;
length = size - this.pos;
}

function afterRead(er, bytesRead) {
if (er) {
return fs.close(fd, function(er2) {
return callback(er);
});
}
var req = new FSReqWrap();
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be even neater to fold the FSReqWrap object into the ReadFileContext object. Something for another time, though.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All the FSReqWrap() object are one use, which means they could be reused for each step. Preventing the need to instantiate more objects. Food for thought.

req.oncomplete = readFileAfterRead;
req.context = this;

if (bytesRead === 0) {
return close();
}
binding.read(fd, buffer, offset, length, -1, req);
};

pos += bytesRead;
if (size !== 0) {
if (pos === size) close();
else read();
} else {
// unknown size, just read until we don't get bytes.
buffers.push(buffer.slice(0, bytesRead));
read();
}
ReadFileContext.prototype.close = function(err) {
var req = new FSReqWrap();
req.oncomplete = readFileAfterClose;
req.context = this;
this.err = err;
binding.close(this.fd, req);
};

function readFileAfterOpen(err, fd) {
var context = this.context;

if (err) {
var callback = context.callback;
callback(err);
return;
}

function close() {
fs.close(fd, function(er) {
if (size === 0) {
// collected the data into the buffers list.
buffer = Buffer.concat(buffers, pos);
} else if (pos < size) {
buffer = buffer.slice(0, pos);
}
context.fd = fd;

if (encoding) buffer = buffer.toString(encoding);
return callback(er, buffer);
});
var req = new FSReqWrap();
req.oncomplete = readFileAfterStat;
req.context = context;
binding.fstat(fd, req);
}

function readFileAfterStat(err, st) {
var context = this.context;

if (err)
return context.close(err);

var size = context.size = st.size;

if (size === 0) {
context.buffers = [];
context.read();
return;
}
};

if (size > kMaxLength) {
err = new RangeError('File size is greater than possible Buffer: ' +
`0x${kMaxLength.toString(16)} bytes`);
return context.close(err);
}

context.buffer = new SlowBuffer(size);
context.read();
}

function readFileAfterRead(err, bytesRead) {
var context = this.context;

if (err)
return context.close(err);

if (bytesRead === 0)
return context.close();

context.pos += bytesRead;

if (context.size !== 0) {
if (context.pos === context.size)
context.close();
else
context.read();
} else {
// unknown size, just read until we don't get bytes.
context.buffers.push(context.buffer.slice(0, bytesRead));
context.read();
}
}

function readFileAfterClose(err) {
var context = this.context;
var buffer = null;
var callback = context.callback;

if (context.err)
return callback(context.err);

if (context.size === 0)
buffer = Buffer.concat(context.buffers, context.pos);
else if (context.pos < context.size)
buffer = context.buffer.slice(0, context.pos);
else
buffer = context.buffer;

if (context.encoding)
buffer = buffer.toString(context.encoding);

callback(err, buffer);
}


fs.readFileSync = function(path, options) {
if (!options) {
Expand Down