diff --git a/src/cli/commands/daemon.js b/src/cli/commands/daemon.js index 076a6d9fbe..751b0b3611 100644 --- a/src/cli/commands/daemon.js +++ b/src/cli/commands/daemon.js @@ -10,11 +10,22 @@ module.exports = { describe: 'Start a long-running daemon process', - handler () { + builder: { + 'enable-sharding-experiment': { + type: 'boolean', + defaultt: false + }, + 'enable-pubsub-experiment': { + type: 'booleam', + default: false + } + }, + + handler (argv) { console.log('Initializing daemon...') const repoPath = utils.getRepoPath() - httpAPI = new HttpAPI(repoPath) + httpAPI = new HttpAPI(process.env.IPFS_PATH, argv) httpAPI.start((err) => { if (err && err.code === 'ENOENT') { diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index e133056c16..af5cad18af 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -8,6 +8,7 @@ const pull = require('pull-stream') const paramap = require('pull-paramap') const zip = require('pull-zip') const toPull = require('stream-to-pull-stream') +const utils = require('../../utils') const WRAPPER = 'wrapper/' @@ -56,6 +57,14 @@ module.exports = { alias: 'w', type: 'boolean', default: false + }, + 'enable-sharding-experiment': { + type: 'boolean', + defaultt: false + }, + 'shard-split-threshold': { + type: 'integer', + default: 1000 } }, @@ -63,7 +72,12 @@ module.exports = { const inPath = checkPath(argv.file, argv.recursive) const index = inPath.lastIndexOf('/') + 1 const options = { - strategy: argv.trickle ? 'trickle' : 'balanced' + strategy: argv.trickle ? 'trickle' : 'balanced', + shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity + } + + if (argv.enableShardingExperiment && utils.isDaemonOn()) { + throw new Error('Error: Enabling the sharding experiment should be done on the daemon') } const ipfs = argv.ipfs diff --git a/src/core/components/files.js b/src/core/components/files.js index b84cff3fef..a5e28e7f7e 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -18,10 +18,14 @@ const Duplex = require('stream').Duplex module.exports = function files (self) { const createAddPullStream = (options) => { + const opts = Object.assign({}, { + shardSplitThreshold: self._options.EXPERIMENTAL.sharding ? 1000 : Infinity + }, options) + return pull( pull.map(normalizeContent), pull.flatten(), - importer(self._ipldResolver, options), + importer(self._ipldResolver, opts), pull.asyncMap(prepareFile.bind(null, self)) ) } diff --git a/src/core/index.js b/src/core/index.js index 042d7731e5..f0b7c0c815 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -97,6 +97,9 @@ class IPFS extends EventEmitter { if (this._options.EXPERIMENTAL.pubsub) { this.log('EXPERIMENTAL pubsub is enabled') } + if (this._options.EXPERIMENTAL.sharding) { + this.log('EXPERIMENTAL sharding is enabled') + } this.state = require('./state')(this) boot(this) diff --git a/src/http-api/index.js b/src/http-api/index.js index 3fb36e73b2..0c93b889e2 100644 --- a/src/http-api/index.js +++ b/src/http-api/index.js @@ -40,7 +40,8 @@ function HttpApi (repo, config) { start: true, config: config, EXPERIMENTAL: { - pubsub: true + pubsub: true, + sharding: config && config.enableShardingExperiment } }) } catch (err) { diff --git a/test/core/files-sharding.spec.js b/test/core/files-sharding.spec.js new file mode 100644 index 0000000000..5c0d2609f6 --- /dev/null +++ b/test/core/files-sharding.spec.js @@ -0,0 +1,93 @@ +/* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) +const pull = require('pull-stream') + +const IPFS = require('../../src/core') +const createTempRepo = require('../utils/create-repo-node.js') + +describe('files dir', () => { + const files = [] + for (let i = 0; i < 1005; i++) { + files.push({ + path: 'test-folder/' + i, + content: new Buffer('some content ' + i) + }) + } + + describe('without sharding', () => { + let ipfs + + before((done) => { + ipfs = new IPFS({ + repo: createTempRepo(), + config: { + Bootstrap: [] + } + }) + ipfs.once('start', done) + }) + + after((done) => { + ipfs.stop(done) + }) + + it('should be able to add dir without sharding', (done) => { + pull( + pull.values(files), + ipfs.files.createAddPullStream(), + pull.collect((err, results) => { + expect(err).to.not.exist() + const last = results[results.length - 1] + expect(last.path).to.be.eql('test-folder') + expect(last.hash).to.be.eql('QmWWM8ZV6GPhqJ46WtKcUaBPNHN5yQaFsKDSQ1RE73w94Q') + done() + }) + ) + + after((done) => { + ipfs.stop(() => done()) // ignore stop errors + }) + }) + }) + + describe('with sharding', () => { + let ipfs + + before((done) => { + ipfs = new IPFS({ + repo: createTempRepo(), + config: { + Bootstrap: [] + }, + EXPERIMENTAL: { + sharding: true + } + }) + ipfs.once('start', done) + }) + + after((done) => { + ipfs.stop(() => done()) // ignore stop errors + }) + + it('should be able to add dir with sharding', (done) => { + pull( + pull.values(files), + ipfs.files.createAddPullStream(), + pull.collect((err, results) => { + expect(err).to.not.exist() + const last = results[results.length - 1] + expect(last.path).to.be.eql('test-folder') + expect(last.hash).to.be.eql('QmZjYC1kWrLmiRYbEmGSo2PEpMixzT2k2xoCKSBzt8KDcy') + done() + }) + ) + }) + }) +})