Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add cloning support #40

Closed
wants to merge 10 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
13 changes: 13 additions & 0 deletions fs.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ const fs = require('graceful-fs');
const makeDir = require('make-dir');
const pEvent = require('p-event');
const CpFileError = require('./cp-file-error');
const {version} = process;

const stat = promisify(fs.stat);
const lstat = promisify(fs.lstat);
Expand Down Expand Up @@ -69,6 +70,18 @@ exports.makeDirSync = path => {
}
};

exports.cloneFileSync = (source, destination, flags) => {
try {
if (!Object.prototype.hasOwnProperty.call(fs.constants, 'COPYFILE_FICLONE_FORCE')) {
throw new CpFileError(`Node ${version} does not understand cloneFile`);
}

fs.copyFileSync(source, destination, flags | fs.constants.COPYFILE_FICLONE_FORCE);
} catch (error) {
throw new CpFileError(`Cannot clone from \`${source}\` to \`${destination}\`: ${error.message}`, error);
}
};

exports.copyFileSync = (source, destination, flags) => {
try {
fs.copyFileSync(source, destination, flags);
Expand Down
7 changes: 7 additions & 0 deletions index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@ declare namespace cpFile {
@default true
*/
readonly overwrite?: boolean;

/**
Perform a fast copy-on-write clone (reflink) if possible.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not clear from the docs here what force does.

Maybe also mention some use-cases for needing to set it to false or 'force' as I cannot think of anything reason to not have this be just true all the time.

Copy link
Author

@Artoria2e5 Artoria2e5 Jul 6, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is analogous to gnu cp's --reflink=auto vs --reflink. From a usability standpoint, it is usually preferable to have a fallback (hence true), but when someone comes onto a new platform it's nice to have a test for whether reflink is supported (hence "force").

@default true
*/
readonly clone?: boolean | 'force';
}

interface ProgressData {
Expand Down
55 changes: 46 additions & 9 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,40 @@ const pEvent = require('p-event');
const CpFileError = require('./cp-file-error');
const fs = require('./fs');
const ProgressEmitter = require('./progress-emitter');
const {version} = process;

const defaultOptions = {
overwrite: true,
clone: true
};

const updateStats = async (source, destination) => {
const stats = await fs.lstat(source);

return Promise.all([
fs.utimes(destination, stats.atime, stats.mtime),
fs.chmod(destination, stats.mode)
]);
};

const cpFileAsync = async (source, destination, options, progressEmitter) => {
let readError;
const stat = await fs.stat(source);
progressEmitter.size = stat.size;

// Try to do a fast-path using FICLONE_FORCE. This will be very fast if at all successful.
if (options.clone) {
try {
fs.cloneFileSync(source, destination, options.overwrite ? null : fsConstants.COPYFILE_EXCL);
progressEmitter.writtenBytes = progressEmitter.size;
return updateStats(source, destination);
} catch (error) {
if (options.clone === 'force') {
throw error;
}
}
}

const readStream = await fs.createReadStream(source);
await fs.makeDir(path.dirname(destination));
const writeStream = fs.createWriteStream(destination, {flags: options.overwrite ? 'w' : 'wx'});
Expand Down Expand Up @@ -40,12 +68,7 @@ const cpFileAsync = async (source, destination, options, progressEmitter) => {
}

if (shouldUpdateStats) {
const stats = await fs.lstat(source);

return Promise.all([
fs.utimes(destination, stats.atime, stats.mtime),
fs.chmod(destination, stats.mode)
]);
return updateStats(source, destination);
}
};

Expand All @@ -55,7 +78,7 @@ const cpFile = (sourcePath, destinationPath, options) => {
}

options = {
overwrite: true,
...defaultOptions,
...options
};

Expand Down Expand Up @@ -88,15 +111,29 @@ module.exports.sync = (source, destination, options) => {
}

options = {
overwrite: true,
...defaultOptions,
...options
};

const stat = fs.statSync(source);
checkSourceIsFile(stat, source);
fs.makeDirSync(path.dirname(destination));

const flags = options.overwrite ? null : fsConstants.COPYFILE_EXCL;
let flags = 0;
if (!options.overwrite) {
flags |= fsConstants.COPYFILE_EXCL;
}

if (options.clone === true) {
flags |= fsConstants.COPYFILE_FICLONE;
} else if (options.clone === 'force') {
if (!Object.prototype.hasOwnProperty.call(fs.constants, 'COPYFILE_FICLONE_FORCE')) {
throw new CpFileError(`Node ${version} does not understand cloneFile`);
Artoria2e5 marked this conversation as resolved.
Show resolved Hide resolved
}

flags |= fsConstants.COPYFILE_FICLONE_FORCE;
}

try {
fs.copyFileSync(source, destination, flags);
} catch (error) {
Expand Down
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
"node": ">=10"
},
"scripts": {
"test": "xo && nyc ava && tsd"
"test": "xo && nyc ava && tsd",
"fix": "xo --fix"
},
"files": [
"cp-file-error.js",
Expand Down
5 changes: 5 additions & 0 deletions progress-emitter.js
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
'use strict';
const EventEmitter = require('events');

/** @type {WeakMap<ProgressEmitter, number>} */
const writtenBytes = new WeakMap();

class ProgressEmitter extends EventEmitter {
constructor(sourcePath, destinationPath) {
super();
/** @type {string} */
this._sourcePath = sourcePath;
/** @type {string} */
this._destinationPath = destinationPath;
/** @type {number | undefined} */
this.size = undefined;
}

get writtenBytes() {
Expand Down
7 changes: 7 additions & 0 deletions readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,13 @@ Default: `true`

Overwrite existing destination file.

##### clone

Type: `boolean | 'force'`\
Default: `true`

Perform a fast copy-on-write clone (reflink) if possible.

### cpFile.on('progress', handler)

Progress reporting. Only available when using the async method.
Expand Down
15 changes: 12 additions & 3 deletions test/async.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,15 @@ import test from 'ava';
import {v4 as uuidv4} from 'uuid';
import sinon from 'sinon';
import assertDateEqual from './helpers/_assert';
import getFsName from './helpers/_whatfs';
import {buildEACCES, buildEIO, buildENOSPC, buildENOENT, buildEPERM, buildERRSTREAMWRITEAFTEREND} from './helpers/_fs-errors';
import cpFile from '..';

const THREE_HUNDRED_KILO = (100 * 3 * 1024) + 1;

test.before(() => {
process.chdir(path.dirname(__dirname));
console.log(`fs info: ${getFsName('.')}`);
});

test.beforeEach(t => {
Expand All @@ -38,20 +40,27 @@ test('reject an Error on missing `destination`', async t => {
});

test('copy a file', async t => {
await cpFile('license', t.context.destination);
await cpFile('license', t.context.destination, {clone: false});
t.is(fs.readFileSync(t.context.destination, 'utf8'), fs.readFileSync('license', 'utf8'));
});

test('copy an empty file', async t => {
fs.writeFileSync(t.context.source, '');
await cpFile(t.context.source, t.context.destination);
await cpFile(t.context.source, t.context.destination, {clone: false});
t.is(fs.readFileSync(t.context.destination, 'utf8'), '');
});

test('copy big files', async t => {
const buffer = crypto.randomBytes(THREE_HUNDRED_KILO);
fs.writeFileSync(t.context.source, buffer);
await cpFile(t.context.source, t.context.destination);
await cpFile(t.context.source, t.context.destination, {clone: false});
t.true(buffer.equals(fs.readFileSync(t.context.destination)));
});

test('clone a big file', async t => {
const buffer = crypto.randomBytes(THREE_HUNDRED_KILO);
fs.writeFileSync(t.context.source, buffer);
await cpFile(t.context.source, t.context.destination, {clone: true});
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can't make this force yet -- I can't figure out how to use different install scripts for different OS. Since cmd is super incompatible with sh I don't think I can get away with some if statements. Otherwise I would be able to make a known-fs disk image and stuff.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, travis on Windows uses mingw sh. Guess I am free to do whatever.

t.true(buffer.equals(fs.readFileSync(t.context.destination)));
});

Expand Down
47 changes: 47 additions & 0 deletions test/helpers/_whatfs.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
'use strict';
const childProcess = require('child_process');

/**
* Returns a Windows drive letter.
* @param {string} path
* @returns {string | undefined}
*/
function driveLetter(path) {
return path.match(/^([A-Z]:)[\\/]/)[1];
}

/**
* Return something with a filesystem name for a path.
* Only works on an English installation for obvious reasons.
* @param {string} path
* @returns {string | undefined}
*/
function getFsNameWin32(path) {
// Get the drive. If it has a drive letter in the front we use it, otherwise assume to be the same as cwd.
const drive = driveLetter(path) || driveLetter(process.cwd);
try {
return childProcess.execSync(`fsutil fsinfo volumeInfo ${drive}`, {encoding: 'ascii'})
.split('\r\n')
.filter(s => s.startsWith('File System Name'))
.join('\r\n');
} catch (_) {
return undefined;
}
}

/**
* Return something with a filesystem name for a path.
* @param {string} path
* @returns {string | undefined}
* @see {@link https://unix.stackexchange.com/a/21807|How can I determine the fs type of my current working directory?}
*/
function getFsNamePosix(path) {
try {
return childProcess.execSync(`mount | grep "^$(df -Pk '${path.replace(/'/g, '\'\\\'\'')}' | head -n 2 | tail -n 1 | cut -f 1 -d ' ') "`,
{encoding: 'utf8'});
} catch (_) {
return undefined;
}
}

export default process.platform === 'win32' ? getFsNameWin32 : getFsNamePosix;