Skip to content

Commit

Permalink
Merge d9ba2aa into bc38ab3
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexpusch
alexpusch committed Sep 23, 2015
2 parents bc38ab3 + d9ba2aa commit 7877946
Show file tree
Hide file tree
Showing 3 changed files with 182 additions and 3 deletions.
14 changes: 14 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,20 @@ subdirectories will be traversed.
polling for binary files.
([see list of binary extensions](https://github.com/sindresorhus/binary-extensions/blob/master/binary-extensions.json))

#### Waiting for write operation to finish
* `waitWriteFinish` (default: `false`).
The `add` event will fire when a file first appear on disk, before the entire
file has been written. Furthermore, in some cases some `change` events will be emitted while the file is being written.
In some cases, especially when watching for large files there will be a need to
wait for the write operation to finish before responding to the file creation.
Setting `waitWriteFinish` to `true` will poll a newly created file size, holding
its `add` and `change` events until its size will not change for a configurable amount of time.
* `writeFinishThreshold` (default: 2000).
Amount of time in milliseconds for a file size to remain constant before emitting its
`add` event. Unfortunately this duration is heavily dependent on OS and local hardware.
For accurate detection this parameter should be relatively high, making file watching much less
responsive. Use with cation.

#### Errors
* `ignorePermissionErrors` (default: `false`). Indicates whether to watch files
that don't have read permissions if possible. If watching fails due to `EPERM`
Expand Down
74 changes: 73 additions & 1 deletion index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,11 @@ function FSWatcher(_opts) {

if (undef('followSymlinks')) opts.followSymlinks = true;

if (undef('waitWriteFinish')) opts.waitWriteFinish = false;
if (undef('writeFinishThreshold')) opts.writeFinishThreshold = 2000;

if (opts.waitWriteFinish) this._pendingWrites = Object.create(null);

this._isntIgnored = function(path, stat) {
return !this._isIgnored(path, stat);
}.bind(this);
Expand Down Expand Up @@ -111,6 +116,9 @@ FSWatcher.prototype._emit = function(event, path, val1, val2, val3) {
if (val3 !== undefined) args.push(val1, val2, val3);
else if (val2 !== undefined) args.push(val1, val2);
else if (val1 !== undefined) args.push(val1);

if ((this.options.waitWriteFinish && this._pendingWrites[path])) return this;

if (this.options.atomic) {
if (event === 'unlink') {
this._pendingUnlinks[path] = args;
Expand All @@ -128,6 +136,7 @@ FSWatcher.prototype._emit = function(event, path, val1, val2, val3) {
}
}


if (event === 'change') {
if (!this._throttle('change', path, 50)) return this;
}
Expand All @@ -137,7 +146,19 @@ FSWatcher.prototype._emit = function(event, path, val1, val2, val3) {
if (event !== 'error') this.emit.apply(this, ['all'].concat(args));
}.bind(this);

if (
if (this.options.waitWriteFinish && event === 'add') {
this._awaitWriteFinish(path, this.options.writeFinishThreshold, function(err, stats){
if(err){
event = args[0] = 'error';
args[1] = err;
emitEvent();
} else if(stats){
// if stats doesn't exist the file must have been deleted
args.push(stats);
emitEvent();
}
});
} else if (
this.options.alwaysStat && val1 === undefined &&
(event === 'add' || event === 'addDir' || event === 'change')
) {
Expand Down Expand Up @@ -194,6 +215,51 @@ FSWatcher.prototype._throttle = function(action, path, timeout) {
return throttled[path];
};

// Private method: Awaits write operation to finish
//
// * path - string, path being acted upon
// * threshold - int, time in milliseconds a file size must be fixed before acknowledgeing write operation is finished
// * callback - function, callback to call when write operation is finished
// Polls a newly created file for size variations. When files size does not change for 'threshold'
// milliseconds calls callback.
FSWatcher.prototype._awaitWriteFinish = function(path, threshold, callback){
var timeoutHandler;

var awaitWriteFinish = function(prevStat){
fs.stat(path, function(err, curStat){
if(err){
// if the file have been erased, the file entry in _pendingWrites will
// be deleted in the unlink event.
if(err.code == 'ENOENT') return;

return callback(err);
}

var now = new Date();
if(this._pendingWrites[path] === undefined){
this._pendingWrites[path] = {
creationTime: now,
cancelWait: function(){
delete this._pendingWrites[path];
clearTimeout(timeoutHandler);
return callback();
}.bind(this)
}
return timeoutHandler = setTimeout(awaitWriteFinish.bind(this, curStat), this.options.interval);
}

if(curStat.size == prevStat.size && now - this._pendingWrites[path].creationTime > threshold){
delete this._pendingWrites[path];
callback(null, curStat);
} else{
return timeoutHandler = setTimeout(awaitWriteFinish.bind(this, curStat), this.options.interval);
}
}.bind(this));
}.bind(this);

awaitWriteFinish();
}

// Private method: Determines whether user has asked to ignore this path
//
// * path - string, path to file or directory
Expand Down Expand Up @@ -368,6 +434,12 @@ FSWatcher.prototype._remove = function(directory, item) {
var wasTracked = parent.has(item);
parent.remove(item);

// If we wait for this file to be fully written, cancel the wait.
if(this.options.waitWriteFinish && this._pendingWrites[path]) {
this._pendingWrites[path].cancelWait();
return;
}

// The Entry will either be a directory that just got removed
// or a bogus entry to a file, in either case we have to remove it
delete this._watched[path];
Expand Down
97 changes: 95 additions & 2 deletions test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ afterEach(function() {
function rmFixtures() {
try { fs.unlinkSync(getFixturePath('link')); } catch(err) {}
try { fs.unlinkSync(getFixturePath('add.txt')); } catch(err) {}
try { fs.unlinkSync(getFixturePath('no-add.txt')); } catch(err) {}
try { fs.unlinkSync(getFixturePath('no-change.txt')); } catch(err) {}
try { fs.unlinkSync(getFixturePath('late-change.txt')); } catch(err) {}
try { fs.unlinkSync(getFixturePath('early-unlinked.txt')); } catch(err) {}
try { fs.unlinkSync(getFixturePath('moved.txt')); } catch(err) {}
try { fs.unlinkSync(getFixturePath('movedagain.txt')); } catch(err) {}
try { fs.unlinkSync(getFixturePath('cantread.txt')); } catch(err) {}
Expand All @@ -53,7 +57,7 @@ after(function() {


describe('chokidar', function() {
this.timeout(3000);
this.timeout(5000);
it('should expose public API methods', function() {
chokidar.FSWatcher.should.be.a('function');
chokidar.watch.should.be.a('function');
Expand Down Expand Up @@ -88,7 +92,7 @@ function runTests(options) {
var intrvl = setInterval(function() {
if (spies.every(isSpyReady)) finish();
}, 5);
var to = setTimeout(finish, 1500);
var to = setTimeout(finish, 3000);
}
function d(fn, quicker, forceTimeout) {
if (options.usePolling || forceTimeout) {
Expand Down Expand Up @@ -1219,6 +1223,95 @@ function runTests(options) {
});
});
});
describe('waitWriteFinish', function() {
beforeEach(function() {
options.waitWriteFinish = true;
options.writeFinishThreshold = 1000;
});
it('should not emit add event before a file is fully written', function(done) {
var spy = sinon.spy();
var testPath = getFixturePath('no-add.txt');
stdWatcher()
.on('all', spy)
.on('ready', function() {
fs.writeFileSync(testPath, 'hello');
dd(function(){
spy.should.not.have.been.calledWith('add');
done();
})();
});
});
it('should wait for the file to be fully written before emiting the add event', function(done) {
var spy = sinon.spy();
var testPath = getFixturePath('add.txt');
stdWatcher()
.on('all', spy)
.on('ready', function() {
fs.writeFileSync(testPath, 'hello');
dd(function(){
spy.should.not.have.been.calledWith('add');
setTimeout(function() {
spy.should.have.been.calledWith('add');
done();
}, 1100);
})();
}.bind(this));
});
it('should not emit change event while a file have not been fully written', function(done) {
var spy = sinon.spy();
var testPath = getFixturePath('no-change.txt');
stdWatcher()
.on('all', spy)
.on('ready', function() {
fs.writeFileSync(testPath, 'hello');
d(function() {
spy.should.not.have.been.calledWith('add', testPath);
fs.writeFileSync(testPath, 'edit');
dd(function() {
spy.should.not.have.been.calledWith('change', testPath);
done();
})();
}());
}.bind(this));
});
it('should emit change event after the file have been fully written', function(done) {
var spy = sinon.spy(), changeSpy = sinon.spy();
var testPath = getFixturePath('late-change.txt');
stdWatcher()
.on('all', spy)
.on('change', changeSpy)
.on('ready', function(){
fs.writeFileSync(testPath, 'hello');
dd(function() {
spy.should.not.have.been.calledWith('add', testPath);
setTimeout(function() {
fs.writeFileSync(testPath, 'edit');
waitFor([changeSpy], function(){
changeSpy.should.have.been.calledWith(testPath);
done();
});
}, 1500);
})();
}.bind(this))
});
it('should not raise any event for a file that was deleted before fully written', function(done) {
var spy = sinon.spy();
var testPath = getFixturePath('early-unlinked.txt');
stdWatcher()
.on('all', spy)
.on('ready', function() {
fs.writeFileSync(testPath, 'hello');
d(function() {
fs.unlinkSync(testPath);
var now = new Date();
setTimeout(function() {
spy.should.not.have.been.calledWith(sinon.match.string, testPath);
done();
}, 1100);
})();
});
});
});
});
describe('unwatch', function() {
beforeEach(function(done) {
Expand Down

0 comments on commit 7877946

Please sign in to comment.