From f0c5aa0aa5a4f49ebc1e186a99a654020e1c6109 Mon Sep 17 00:00:00 2001 From: Rasmus Erik Voel Jensen Date: Tue, 1 Aug 2017 12:37:52 +0200 Subject: [PATCH 01/11] feat: cli ls (#927) License: MIT Signed-off-by: Rasmus Erik Voel Jensen --- src/cli/commands/ls.js | 60 ++++++++++++++++++++++++++++++++++++++++++ src/cli/utils.js | 8 ++++++ test/cli/commands.js | 2 +- test/cli/files.js | 40 ++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 src/cli/commands/ls.js diff --git a/src/cli/commands/ls.js b/src/cli/commands/ls.js new file mode 100644 index 0000000000..f17afaa415 --- /dev/null +++ b/src/cli/commands/ls.js @@ -0,0 +1,60 @@ +'use strict' + +const {print, rightpad} = require('../utils') +const Unixfs = require('ipfs-unixfs') + +module.exports = { + command: 'ls ', + + describe: 'List files for the given directory', + + builder: { + v: { + alias: 'headers', + desc: 'Print table headers (Hash, Size, Name).', + type: 'boolean', + default: false + }, + 'resolve-type': { + desc: 'Resolve linked objects to find out their types. (not implemented yet)', + type: 'boolean', + default: false // should be true when implemented + } + }, + + handler (argv) { + let path = argv.key + if (path.startsWith('/ipfs/')) { + path = path.replace('/ipfs/', '') + } + + argv.ipfs.object.get(path, {enc: 'base58'}, (err, node) => { + if (err) { + throw err + } + let {data, links} = node.toJSON() + + const fileDesc = Unixfs.unmarshal(data) + if (fileDesc.type !== 'directory') { + throw new Error('merkeldag node was not a directory') // TODO: support shards + } + + if (argv['resolve-type']) { + throw new Error('--resolve-type not implemented yet') + } + + if (argv.headers) { + links = [{multihash: 'Hash', size: 'Size', name: 'Name'}].concat(links) + } + + const multihashWidth = Math.max.apply(null, links.map((file) => String(file.multihash).length)) + const sizeWidth = Math.max.apply(null, links.map((file) => String(file.size).length)) + + links.forEach((file) => { + print(rightpad(file.multihash, multihashWidth + 1) + + rightpad(file.size, sizeWidth + 1) + + file.name) + }) + }) + } +} diff --git a/src/cli/utils.js b/src/cli/utils.js index 4565fc316e..d44bf4b4fd 100644 --- a/src/cli/utils.js +++ b/src/cli/utils.js @@ -100,3 +100,11 @@ exports.createProgressBar = (totalBytes) => { total: totalBytes }) } + +exports.rightpad = (val, n) => { + let result = String(val) + for (let i = result.length; i < n; ++i) { + result += ' ' + } + return result +} diff --git a/test/cli/commands.js b/test/cli/commands.js index 94ffb4aad4..e6d9059d36 100644 --- a/test/cli/commands.js +++ b/test/cli/commands.js @@ -4,7 +4,7 @@ const expect = require('chai').expect const runOnAndOff = require('../utils/on-and-off') -const commandCount = 56 +const commandCount = 57 describe('commands', () => runOnAndOff((thing) => { let ipfs diff --git a/test/cli/files.js b/test/cli/files.js index c07762e1f3..1420a63620 100644 --- a/test/cli/files.js +++ b/test/cli/files.js @@ -254,6 +254,46 @@ describe('files', () => runOnAndOff((thing) => { }) }) + it('ls', () => { + return ipfs('ls QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2') + .then((out) => { + expect(out).to.eql( + 'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9 123530 blocks\n' + + 'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN 3939 config\n' + + 'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz 5503 datastore\n' + + 'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU 7397 init-docs\n' + + 'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV 10 version\n') + }) + }) + + it('ls -v', () => { + return ipfs('ls /ipfs/QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2 -v') + .then((out) => { + expect(out).to.eql( + 'Hash Size Name\n' + + 'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9 123530 blocks\n' + + 'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN 3939 config\n' + + 'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz 5503 datastore\n' + + 'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU 7397 init-docs\n' + + 'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV 10 version\n') + }) + }) + + it('ls --help', () => { + return ipfs('ls --help') + .then((out) => { + expect(out.split('\n').slice(1)).to.eql(['', + 'Options:', + ' -q, --quiet suppress output [boolean]', + ' --help Show help [boolean]', + ' -v, --headers Print table headers (Hash, Size, Name).', + ' [boolean] [default: false]', + ' --resolve-type Resolve linked objects to find out their types. (not', + ' implemented yet) [boolean] [default: false]', + '', '']) + }) + }) + it('get', () => { return ipfs('files get QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') .then((out) => { From b5b20fb6f4770346363a5d58908158a8bba684d6 Mon Sep 17 00:00:00 2001 From: Rasmus Erik Voel Jensen Date: Wed, 23 Aug 2017 11:17:36 +0200 Subject: [PATCH 02/11] not using destructuring --- src/cli/commands/ls.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli/commands/ls.js b/src/cli/commands/ls.js index f17afaa415..705cd3a8a7 100644 --- a/src/cli/commands/ls.js +++ b/src/cli/commands/ls.js @@ -1,6 +1,6 @@ 'use strict' -const {print, rightpad} = require('../utils') +const utils = require('../utils') const Unixfs = require('ipfs-unixfs') module.exports = { @@ -51,8 +51,8 @@ module.exports = { const sizeWidth = Math.max.apply(null, links.map((file) => String(file.size).length)) links.forEach((file) => { - print(rightpad(file.multihash, multihashWidth + 1) + - rightpad(file.size, sizeWidth + 1) + + utils.print(utils.rightpad(file.multihash, multihashWidth + 1) + + utils.rightpad(file.size, sizeWidth + 1) + file.name) }) }) From 4d4be30e431fafde0483b91b061df4a1bccb448e Mon Sep 17 00:00:00 2001 From: Pedro Teixeira Date: Sun, 12 Nov 2017 11:05:08 +0000 Subject: [PATCH 03/11] files ls --- src/cli/bin.js | 3 +- src/cli/commands/files/ls.js | 91 ++++++++++++++++++++++++++++++++++++ src/cli/commands/ls.js | 60 ------------------------ src/core/components/files.js | 9 +++- 4 files changed, 101 insertions(+), 62 deletions(-) create mode 100644 src/cli/commands/files/ls.js delete mode 100644 src/cli/commands/ls.js diff --git a/src/cli/bin.js b/src/cli/bin.js index c9a72f2aa1..9e9c18fed9 100755 --- a/src/cli/bin.js +++ b/src/cli/bin.js @@ -36,7 +36,8 @@ const cli = yargs const addCmd = require('./commands/files/add') const catCmd = require('./commands/files/cat') const getCmd = require('./commands/files/get') -const aliases = [addCmd, catCmd, getCmd] +const lsCmd = require('./commands/files/ls') +const aliases = [addCmd, catCmd, getCmd, lsCmd] aliases.forEach((alias) => { cli.command(alias.command, alias.describe, alias.builder, alias.handler) }) diff --git a/src/cli/commands/files/ls.js b/src/cli/commands/files/ls.js new file mode 100644 index 0000000000..70810ecd3a --- /dev/null +++ b/src/cli/commands/files/ls.js @@ -0,0 +1,91 @@ +'use strict' + +const utils = require('../../utils') +const Unixfs = require('ipfs-unixfs') +const pull = require('pull-stream') + +module.exports = { + command: 'ls ', + + describe: 'List files for the given directory', + + builder: { + v: { + alias: 'headers', + desc: 'Print table headers (Hash, Size, Name).', + type: 'boolean', + default: false + }, + 'resolve-type': { + desc: 'Resolve linked objects to find out their types. (not implemented yet)', + type: 'boolean', + default: false // should be true when implemented + } + }, + + handler (argv) { + let path = argv.key + if (path.startsWith('/ipfs/')) { + path = path.replace('/ipfs/', '') + } + + pull( + argv.ipfs.files.ls(path, 0), + pull.collect((err, links) => { + if (err) { + throw err + } + + if (argv.headers) { + links = [{hash: 'Hash', size: 'Size', name: 'Name'}].concat(links) + } + + links = links.filter((link) => link.path !== path) + links.forEach((link) => { + if (link.type === 'dir') { + // directory: add trailing "/" + link.name = (link.name || '') + '/' + } + }) + const multihashWidth = Math.max.apply(null, links.map((file) => file.hash.length)) + const sizeWidth = Math.max.apply(null, links.map((file) => String(file.size).length)) + + links.forEach((file) => { + utils.print(utils.rightpad(file.hash, multihashWidth + 1) + + utils.rightpad(file.size || '', sizeWidth + 1) + + file.name) + }) + }) + ) + + + // argv.ipfs.object.get(path, {enc: 'base58'}, (err, node) => { + // if (err) { + // throw err + // } + // let {data, links} = node.toJSON() + + // const fileDesc = Unixfs.unmarshal(data) + // if (fileDesc.type !== 'directory') { + // throw new Error('merkeldag node was not a directory') // TODO: support shards + // } + + // if (argv['resolve-type']) { + // throw new Error('--resolve-type not implemented yet') + // } + + // if (argv.headers) { + // links = [{multihash: 'Hash', size: 'Size', name: 'Name'}].concat(links) + // } + + // const multihashWidth = Math.max.apply(null, links.map((file) => String(file.multihash).length)) + // const sizeWidth = Math.max.apply(null, links.map((file) => String(file.size).length)) + + // links.forEach((file) => { + // utils.print(utils.rightpad(file.multihash, multihashWidth + 1) + + // utils.rightpad(file.size, sizeWidth + 1) + + // file.name) + // }) + // }) + } +} diff --git a/src/cli/commands/ls.js b/src/cli/commands/ls.js deleted file mode 100644 index 705cd3a8a7..0000000000 --- a/src/cli/commands/ls.js +++ /dev/null @@ -1,60 +0,0 @@ -'use strict' - -const utils = require('../utils') -const Unixfs = require('ipfs-unixfs') - -module.exports = { - command: 'ls ', - - describe: 'List files for the given directory', - - builder: { - v: { - alias: 'headers', - desc: 'Print table headers (Hash, Size, Name).', - type: 'boolean', - default: false - }, - 'resolve-type': { - desc: 'Resolve linked objects to find out their types. (not implemented yet)', - type: 'boolean', - default: false // should be true when implemented - } - }, - - handler (argv) { - let path = argv.key - if (path.startsWith('/ipfs/')) { - path = path.replace('/ipfs/', '') - } - - argv.ipfs.object.get(path, {enc: 'base58'}, (err, node) => { - if (err) { - throw err - } - let {data, links} = node.toJSON() - - const fileDesc = Unixfs.unmarshal(data) - if (fileDesc.type !== 'directory') { - throw new Error('merkeldag node was not a directory') // TODO: support shards - } - - if (argv['resolve-type']) { - throw new Error('--resolve-type not implemented yet') - } - - if (argv.headers) { - links = [{multihash: 'Hash', size: 'Size', name: 'Name'}].concat(links) - } - - const multihashWidth = Math.max.apply(null, links.map((file) => String(file.multihash).length)) - const sizeWidth = Math.max.apply(null, links.map((file) => String(file.size).length)) - - links.forEach((file) => { - utils.print(utils.rightpad(file.multihash, multihashWidth + 1) + - utils.rightpad(file.size, sizeWidth + 1) + - file.name) - }) - }) - } -} diff --git a/src/core/components/files.js b/src/core/components/files.js index e2f6443350..c1f7731ae8 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -13,6 +13,7 @@ const waterfall = require('async/waterfall') const isStream = require('is-stream') const Duplex = require('stream').Duplex const CID = require('cids') +const toB58String = require('multihashes').toB58String module.exports = function files (self) { const createAddPullStream = (options) => { @@ -118,7 +119,13 @@ module.exports = function files (self) { getPull: promisify((ipfsPath, callback) => { callback(null, exporter(ipfsPath, self._ipldResolver)) - }) + }), + + ls: (ipfsPath) => pull( + exporter(ipfsPath, self._ipldResolver, { maxDepth: 1 }), + pull.filter((node) => node.depth === 1), + pull.map((node) => Object.assign({}, node, { hash: toB58String(node.hash) })), + ) } } From 1d21f66522a3231b0fc8ec0107e459df1b9cda2d Mon Sep 17 00:00:00 2001 From: Pedro Teixeira Date: Sun, 12 Nov 2017 12:47:31 +0000 Subject: [PATCH 04/11] files.ls exposes a standard stream now --- src/cli/commands/files/ls.js | 72 +++++++++--------------------------- src/core/components/files.js | 4 +- 2 files changed, 20 insertions(+), 56 deletions(-) diff --git a/src/cli/commands/files/ls.js b/src/cli/commands/files/ls.js index 70810ecd3a..988a912a02 100644 --- a/src/cli/commands/files/ls.js +++ b/src/cli/commands/files/ls.js @@ -3,6 +3,7 @@ const utils = require('../../utils') const Unixfs = require('ipfs-unixfs') const pull = require('pull-stream') +const concat = require('concat-stream') module.exports = { command: 'ls ', @@ -29,63 +30,26 @@ module.exports = { path = path.replace('/ipfs/', '') } - pull( - argv.ipfs.files.ls(path, 0), - pull.collect((err, links) => { - if (err) { - throw err - } + argv.ipfs.files.ls(path, 0).pipe(concat((links) => { + if (argv.headers) { + links = [{hash: 'Hash', size: 'Size', name: 'Name'}].concat(links) + } - if (argv.headers) { - links = [{hash: 'Hash', size: 'Size', name: 'Name'}].concat(links) + links = links.filter((link) => link.path !== path) + links.forEach((link) => { + if (link.type === 'dir') { + // directory: add trailing "/" + link.name = (link.name || '') + '/' } - - links = links.filter((link) => link.path !== path) - links.forEach((link) => { - if (link.type === 'dir') { - // directory: add trailing "/" - link.name = (link.name || '') + '/' - } - }) - const multihashWidth = Math.max.apply(null, links.map((file) => file.hash.length)) - const sizeWidth = Math.max.apply(null, links.map((file) => String(file.size).length)) - - links.forEach((file) => { - utils.print(utils.rightpad(file.hash, multihashWidth + 1) + - utils.rightpad(file.size || '', sizeWidth + 1) + - file.name) - }) }) - ) - - - // argv.ipfs.object.get(path, {enc: 'base58'}, (err, node) => { - // if (err) { - // throw err - // } - // let {data, links} = node.toJSON() + const multihashWidth = Math.max.apply(null, links.map((file) => file.hash.length)) + const sizeWidth = Math.max.apply(null, links.map((file) => String(file.size).length)) - // const fileDesc = Unixfs.unmarshal(data) - // if (fileDesc.type !== 'directory') { - // throw new Error('merkeldag node was not a directory') // TODO: support shards - // } - - // if (argv['resolve-type']) { - // throw new Error('--resolve-type not implemented yet') - // } - - // if (argv.headers) { - // links = [{multihash: 'Hash', size: 'Size', name: 'Name'}].concat(links) - // } - - // const multihashWidth = Math.max.apply(null, links.map((file) => String(file.multihash).length)) - // const sizeWidth = Math.max.apply(null, links.map((file) => String(file.size).length)) - - // links.forEach((file) => { - // utils.print(utils.rightpad(file.multihash, multihashWidth + 1) + - // utils.rightpad(file.size, sizeWidth + 1) + - // file.name) - // }) - // }) + links.forEach((file) => { + utils.print(utils.rightpad(file.hash, multihashWidth + 1) + + utils.rightpad(file.size || '', sizeWidth + 1) + + file.name) + }) + })) } } diff --git a/src/core/components/files.js b/src/core/components/files.js index c1f7731ae8..41cbcb0846 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -121,11 +121,11 @@ module.exports = function files (self) { callback(null, exporter(ipfsPath, self._ipldResolver)) }), - ls: (ipfsPath) => pull( + ls: (ipfsPath) => toStream.source(pull( exporter(ipfsPath, self._ipldResolver, { maxDepth: 1 }), pull.filter((node) => node.depth === 1), pull.map((node) => Object.assign({}, node, { hash: toB58String(node.hash) })), - ) + )) } } From f9ee120a4a33d53f56d9ebdd7e6b21346460d656 Mon Sep 17 00:00:00 2001 From: Pedro Teixeira Date: Sun, 12 Nov 2017 12:53:59 +0000 Subject: [PATCH 05/11] added missing package --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 0865299afd..7e7bce7687 100644 --- a/package.json +++ b/package.json @@ -96,6 +96,7 @@ "boom": "^7.1.1", "byteman": "^1.3.5", "cids": "^0.5.2", + "concat-stream": "^1.6.0", "debug": "^3.1.0", "file-type": "^7.2.0", "filesize": "^3.5.11", From 6ccac28a0df6a85231b48d183ca39be37e690df1 Mon Sep 17 00:00:00 2001 From: Pedro Teixeira Date: Sun, 12 Nov 2017 16:08:30 +0000 Subject: [PATCH 06/11] internal lsPull and options for js-ipfs-api compat --- src/cli/commands/files/ls.js | 4 +-- src/core/components/files.js | 24 +++++++++++++---- src/http/api/resources/files.js | 47 +++++++++++++++++++++++++++++++++ src/http/api/routes/files.js | 12 +++++++++ 4 files changed, 80 insertions(+), 7 deletions(-) diff --git a/src/cli/commands/files/ls.js b/src/cli/commands/files/ls.js index 988a912a02..71a817f5eb 100644 --- a/src/cli/commands/files/ls.js +++ b/src/cli/commands/files/ls.js @@ -30,7 +30,7 @@ module.exports = { path = path.replace('/ipfs/', '') } - argv.ipfs.files.ls(path, 0).pipe(concat((links) => { + argv.ipfs.files.ls(path, {}).then((stream) => stream.pipe(concat((links) => { if (argv.headers) { links = [{hash: 'Hash', size: 'Size', name: 'Name'}].concat(links) } @@ -50,6 +50,6 @@ module.exports = { utils.rightpad(file.size || '', sizeWidth + 1) + file.name) }) - })) + }))) } } diff --git a/src/core/components/files.js b/src/core/components/files.js index 41cbcb0846..29d2e17322 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -121,11 +121,25 @@ module.exports = function files (self) { callback(null, exporter(ipfsPath, self._ipldResolver)) }), - ls: (ipfsPath) => toStream.source(pull( - exporter(ipfsPath, self._ipldResolver, { maxDepth: 1 }), - pull.filter((node) => node.depth === 1), - pull.map((node) => Object.assign({}, node, { hash: toB58String(node.hash) })), - )) + lsPull: promisify((ipfsPath, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + callback(null, pull( + exporter(ipfsPath, self._ipldResolver, { maxDepth: 1 }), + pull.filter((node) => node.depth === 1), + pull.map((node) => Object.assign({}, node, { hash: toB58String(node.hash) })) + )) + }), + + ls: promisify((ipfsPath, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + self.files.lsPull(ipfsPath, options).then((stream) => callback(null, toStream.source(stream))) + }) } } diff --git a/src/http/api/resources/files.js b/src/http/api/resources/files.js index 1afbe2c6ef..604618e898 100644 --- a/src/http/api/resources/files.js +++ b/src/http/api/resources/files.js @@ -274,3 +274,50 @@ exports.add = { ) } } + + +exports.ls = { + // uses common parseKey method that returns a `key` + parseArgs: exports.parseKey, + + // main route handler which is called after the above `parseArgs`, but only if the args were valid + handler: (request, reply) => { + const key = request.pre.args.key + const ipfs = request.server.app.ipfs + + ipfs.files.lsPull(key).then((lsStream) => { + const stream = toStream.source(pull( + lsStream, + ndjson.serialize(), + )) + + if (!stream._read) { + stream._read = () => {} + stream._readableState = {} + stream.unpipe = () => {} + } + + stream.on('error', onStreamError) + + reply((stream)) + .header('x-chunked-output', '1') + .header('content-type', 'application/json') + .header('Trailer', 'X-Stream-Error') + + function onStreamError(err) { + console.error(err) + const msg = JSON.stringify({ Message: err.msg, Code: 0 }) + request.raw.res.addTrailers({ + 'X-Stream-Error': msg + }) + } + }).catch(onError) + + function onError(err) { + reply({ + Message: 'Failed to get file: ' + err.message, + Code: 0 + }).code(500) + } + } +} diff --git a/src/http/api/routes/files.js b/src/http/api/routes/files.js index e7ce4f456d..9d9442e293 100644 --- a/src/http/api/routes/files.js +++ b/src/http/api/routes/files.js @@ -42,4 +42,16 @@ module.exports = (server) => { validate: resources.files.add.validate } }) + + api.route({ + // TODO fix method + method: '*', + path: '/api/v0/files/ls', + config: { + pre: [ + { method: resources.files.ls.parseArgs, assign: 'args' } + ], + handler: resources.files.ls.handler + } + }) } From 902449d15e9fb76b28c6d6f1844ac181276468f7 Mon Sep 17 00:00:00 2001 From: Pedro Teixeira Date: Sun, 12 Nov 2017 16:56:15 +0000 Subject: [PATCH 07/11] fixed test output --- test/cli/files.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/test/cli/files.js b/test/cli/files.js index 1420a63620..c61750df82 100644 --- a/test/cli/files.js +++ b/test/cli/files.js @@ -258,10 +258,10 @@ describe('files', () => runOnAndOff((thing) => { return ipfs('ls QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2') .then((out) => { expect(out).to.eql( - 'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9 123530 blocks\n' + + 'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9 123530 blocks/\n' + 'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN 3939 config\n' + - 'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz 5503 datastore\n' + - 'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU 7397 init-docs\n' + + 'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz 5503 datastore/\n' + + 'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU 7397 init-docs/\n' + 'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV 10 version\n') }) }) @@ -271,10 +271,10 @@ describe('files', () => runOnAndOff((thing) => { .then((out) => { expect(out).to.eql( 'Hash Size Name\n' + - 'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9 123530 blocks\n' + + 'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9 123530 blocks/\n' + 'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN 3939 config\n' + - 'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz 5503 datastore\n' + - 'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU 7397 init-docs\n' + + 'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz 5503 datastore/\n' + + 'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU 7397 init-docs/\n' + 'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV 10 version\n') }) }) @@ -283,8 +283,11 @@ describe('files', () => runOnAndOff((thing) => { return ipfs('ls --help') .then((out) => { expect(out.split('\n').slice(1)).to.eql(['', + 'List files for the given directory', + '', 'Options:', - ' -q, --quiet suppress output [boolean]', + ' -v, --version Show version number [boolean]', + ' --silent Show no output. [boolean]', ' --help Show help [boolean]', ' -v, --headers Print table headers (Hash, Size, Name).', ' [boolean] [default: false]', From e70149299708557f5a647375bde88f81a21c56c0 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 13 Nov 2017 10:39:03 +0000 Subject: [PATCH 08/11] chore: update deps --- package.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 7e7bce7687..ed0f8e9866 100644 --- a/package.json +++ b/package.json @@ -75,8 +75,8 @@ "form-data": "^2.3.1", "gulp": "^3.9.1", "hat": "0.0.3", - "interface-ipfs-core": "~0.33.2", - "ipfsd-ctl": "~0.24.0", + "interface-ipfs-core": "~0.34.2", + "ipfsd-ctl": "~0.24.1", "left-pad": "^1.1.3", "lodash": "^4.17.4", "mocha": "^4.0.1", @@ -106,18 +106,18 @@ "hapi": "^16.6.2", "hapi-set-header": "^1.0.2", "hoek": "^5.0.2", - "ipfs-api": "^15.0.1", + "ipfs-api": "^15.0.2", "ipfs-bitswap": "~0.17.4", "ipfs-block": "~0.6.1", "ipfs-block-service": "~0.13.0", "ipfs-multipart": "~0.1.0", "ipfs-repo": "~0.18.3", "ipfs-unixfs": "~0.1.14", - "ipfs-unixfs-engine": "~0.23.1", + "ipfs-unixfs-engine": "~0.24.1", "ipld-resolver": "~0.14.1", "is-ipfs": "^0.3.2", "is-stream": "^1.1.0", - "joi": "^13.0.1", + "joi": "^13.0.2", "libp2p": "~0.13.1", "libp2p-circuit": "~0.1.4", "libp2p-floodsub": "~0.11.1", From 1f3b1cd70e6292ffa7bb7a97d4fd57d8794525fa Mon Sep 17 00:00:00 2001 From: Pedro Teixeira Date: Mon, 13 Nov 2017 18:41:21 +0000 Subject: [PATCH 09/11] ipfs.files.ls -> ipfs.ls + adapting core interface to HTTP API --- package.json | 1 - src/cli/bin.js | 3 +- src/cli/commands/{files => }/ls.js | 11 ++++-- src/core/components/files.js | 27 +++++-------- src/core/index.js | 3 ++ src/http/api/resources/files.js | 61 ++++++++++++++---------------- src/http/api/routes/files.js | 6 +-- 7 files changed, 53 insertions(+), 59 deletions(-) rename src/cli/commands/{files => }/ls.js (89%) diff --git a/package.json b/package.json index ed0f8e9866..c8741be520 100644 --- a/package.json +++ b/package.json @@ -96,7 +96,6 @@ "boom": "^7.1.1", "byteman": "^1.3.5", "cids": "^0.5.2", - "concat-stream": "^1.6.0", "debug": "^3.1.0", "file-type": "^7.2.0", "filesize": "^3.5.11", diff --git a/src/cli/bin.js b/src/cli/bin.js index 9e9c18fed9..c9a72f2aa1 100755 --- a/src/cli/bin.js +++ b/src/cli/bin.js @@ -36,8 +36,7 @@ const cli = yargs const addCmd = require('./commands/files/add') const catCmd = require('./commands/files/cat') const getCmd = require('./commands/files/get') -const lsCmd = require('./commands/files/ls') -const aliases = [addCmd, catCmd, getCmd, lsCmd] +const aliases = [addCmd, catCmd, getCmd] aliases.forEach((alias) => { cli.command(alias.command, alias.describe, alias.builder, alias.handler) }) diff --git a/src/cli/commands/files/ls.js b/src/cli/commands/ls.js similarity index 89% rename from src/cli/commands/files/ls.js rename to src/cli/commands/ls.js index 71a817f5eb..61a19e760c 100644 --- a/src/cli/commands/files/ls.js +++ b/src/cli/commands/ls.js @@ -1,9 +1,8 @@ 'use strict' -const utils = require('../../utils') +const utils = require('../utils') const Unixfs = require('ipfs-unixfs') const pull = require('pull-stream') -const concat = require('concat-stream') module.exports = { command: 'ls ', @@ -30,7 +29,11 @@ module.exports = { path = path.replace('/ipfs/', '') } - argv.ipfs.files.ls(path, {}).then((stream) => stream.pipe(concat((links) => { + argv.ipfs.ls(path, (err, links) => { + if (err) { + throw err + } + if (argv.headers) { links = [{hash: 'Hash', size: 'Size', name: 'Name'}].concat(links) } @@ -50,6 +53,6 @@ module.exports = { utils.rightpad(file.size || '', sizeWidth + 1) + file.name) }) - }))) + }) } } diff --git a/src/core/components/files.js b/src/core/components/files.js index 29d2e17322..0f7fa9509a 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -121,25 +121,18 @@ module.exports = function files (self) { callback(null, exporter(ipfsPath, self._ipldResolver)) }), - lsPull: promisify((ipfsPath, options, callback) => { - if (typeof options === 'function') { - callback = options - options = {} - } - callback(null, pull( - exporter(ipfsPath, self._ipldResolver, { maxDepth: 1 }), - pull.filter((node) => node.depth === 1), - pull.map((node) => Object.assign({}, node, { hash: toB58String(node.hash) })) - )) + immutableLs: promisify((ipfsPath, callback) => { + pull( + self.files.immutableLsPullStream(ipfsPath), + pull.collect(callback)) }), - ls: promisify((ipfsPath, options, callback) => { - if (typeof options === 'function') { - callback = options - options = {} - } - self.files.lsPull(ipfsPath, options).then((stream) => callback(null, toStream.source(stream))) - }) + immutableLsPullStream: (ipfsPath) => { + return pull( + exporter(ipfsPath, self._ipldResolver, { maxDepth: 1 }), + pull.filter((node) => node.depth === 1), + pull.map((node) => Object.assign({}, node, { hash: toB58String(node.hash) }))) + } } } diff --git a/src/core/index.js b/src/core/index.js index 1b85e38291..899f827ec6 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -112,6 +112,9 @@ class IPFS extends EventEmitter { this.state = require('./state')(this) + // ipfs.ls + this.ls = this.files.immutableLs + boot(this) } } diff --git a/src/http/api/resources/files.js b/src/http/api/resources/files.js index 604618e898..b11c6a832f 100644 --- a/src/http/api/resources/files.js +++ b/src/http/api/resources/files.js @@ -276,7 +276,7 @@ exports.add = { } -exports.ls = { +exports.immutableLs = { // uses common parseKey method that returns a `key` parseArgs: exports.parseKey, @@ -285,39 +285,36 @@ exports.ls = { const key = request.pre.args.key const ipfs = request.server.app.ipfs - ipfs.files.lsPull(key).then((lsStream) => { - const stream = toStream.source(pull( - lsStream, - ndjson.serialize(), - )) - - if (!stream._read) { - stream._read = () => {} - stream._readableState = {} - stream.unpipe = () => {} - } - - stream.on('error', onStreamError) - - reply((stream)) - .header('x-chunked-output', '1') - .header('content-type', 'application/json') - .header('Trailer', 'X-Stream-Error') - - function onStreamError(err) { - console.error(err) - const msg = JSON.stringify({ Message: err.msg, Code: 0 }) - request.raw.res.addTrailers({ - 'X-Stream-Error': msg - }) + ipfs.ls(key, (err, files) => { + if (err) { + reply({ + Message: 'Failed to list dir: ' + err.message, + Code: 0 + }).code(500) } - }).catch(onError) - function onError(err) { reply({ - Message: 'Failed to get file: ' + err.message, - Code: 0 - }).code(500) - } + Objects: [{ + Hash: key, + Links: files.map((file) => ({ + Name: file.name, + Hash: file.hash, + Size: file.size, + Type: toTypeCode(file.type) + })) + }] + }) + }) } } + +function toTypeCode(type) { + switch (type) { + case 'dir': + return 1 + case 'file': + return 2 + default: + return 0 + } +} \ No newline at end of file diff --git a/src/http/api/routes/files.js b/src/http/api/routes/files.js index 9d9442e293..2400f6012b 100644 --- a/src/http/api/routes/files.js +++ b/src/http/api/routes/files.js @@ -46,12 +46,12 @@ module.exports = (server) => { api.route({ // TODO fix method method: '*', - path: '/api/v0/files/ls', + path: '/api/v0/ls', config: { pre: [ - { method: resources.files.ls.parseArgs, assign: 'args' } + { method: resources.files.immutableLs.parseArgs, assign: 'args' } ], - handler: resources.files.ls.handler + handler: resources.files.immutableLs.handler } }) } From d72644e52ca91e5cccbb7f3600cfaae63feb8709 Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 14 Nov 2017 09:05:37 +0000 Subject: [PATCH 10/11] chore: update deps --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index c8741be520..ff2b7571a4 100644 --- a/package.json +++ b/package.json @@ -75,7 +75,7 @@ "form-data": "^2.3.1", "gulp": "^3.9.1", "hat": "0.0.3", - "interface-ipfs-core": "~0.34.2", + "interface-ipfs-core": "~0.34.3", "ipfsd-ctl": "~0.24.1", "left-pad": "^1.1.3", "lodash": "^4.17.4", @@ -105,7 +105,7 @@ "hapi": "^16.6.2", "hapi-set-header": "^1.0.2", "hoek": "^5.0.2", - "ipfs-api": "^15.0.2", + "ipfs-api": "^15.1.0", "ipfs-bitswap": "~0.17.4", "ipfs-block": "~0.6.1", "ipfs-block-service": "~0.13.0", @@ -157,7 +157,7 @@ "readable-stream": "2.3.3", "safe-buffer": "^5.1.1", "stream-to-pull-stream": "^1.7.2", - "tar-stream": "^1.5.4", + "tar-stream": "^1.5.5", "temp": "~0.8.3", "through2": "^2.0.3", "update-notifier": "^2.3.0", From bf110e2440a5bfce562c180bb0f99c157f1bc8df Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 14 Nov 2017 09:34:19 +0000 Subject: [PATCH 11/11] fix linting --- src/http/api/resources/files.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/http/api/resources/files.js b/src/http/api/resources/files.js index b11c6a832f..ebb1353f35 100644 --- a/src/http/api/resources/files.js +++ b/src/http/api/resources/files.js @@ -275,7 +275,6 @@ exports.add = { } } - exports.immutableLs = { // uses common parseKey method that returns a `key` parseArgs: exports.parseKey, @@ -308,7 +307,7 @@ exports.immutableLs = { } } -function toTypeCode(type) { +function toTypeCode (type) { switch (type) { case 'dir': return 1 @@ -317,4 +316,5 @@ function toTypeCode(type) { default: return 0 } -} \ No newline at end of file +} +