From 917a564c0d990dfd35d4615436fc8e3609c72a76 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Fri, 24 Feb 2023 13:48:06 +0000 Subject: [PATCH] fix: add methods to import data (#3) In order to have a more consistent developer experience, wrap the `ipfs-unixfs-importer` methods to allow adding data from the `UnixFS` interface. --- src/commands/add.ts | 34 ++++ src/index.ts | 384 +++++++++++++++++++++++++++++++++++++++++++- test/add.spec.ts | 110 +++++++++++++ 3 files changed, 525 insertions(+), 3 deletions(-) create mode 100644 src/commands/add.ts create mode 100644 test/add.spec.ts diff --git a/src/commands/add.ts b/src/commands/add.ts new file mode 100644 index 0000000..e37e95e --- /dev/null +++ b/src/commands/add.ts @@ -0,0 +1,34 @@ +import type { CID } from 'multiformats/cid' +import type { Blockstore } from 'interface-blockstore' +import { ByteStream, DirectoryCandidate, FileCandidate, importBytes, importByteStream, ImportCandidateStream, importDirectory, importer, ImporterOptions, importFile, ImportResult } from 'ipfs-unixfs-importer' + +export async function * addAll (source: ImportCandidateStream, blockstore: Blockstore, options: Partial = {}): AsyncGenerator { + yield * importer(source, blockstore, options) +} + +export async function addBytes (bytes: Uint8Array, blockstore: Blockstore, options: Partial = {}): Promise { + const { cid } = await importBytes(bytes, blockstore, options) + + return cid +} + +export async function addByteStream (bytes: ByteStream, blockstore: Blockstore, options: Partial = {}): Promise { + const { cid } = await importByteStream(bytes, blockstore, options) + + return cid +} + +export async function addFile (file: FileCandidate, blockstore: Blockstore, options: Partial = {}): Promise { + const { cid } = await importFile(file, blockstore, options) + + return cid +} + +export async function addDirectory (dir: Partial, blockstore: Blockstore, options: Partial = {}): Promise { + const { cid } = await importDirectory({ + ...dir, + path: dir.path ?? '-' + }, blockstore, options) + + return cid +} diff --git a/src/index.ts b/src/index.ts index a2c6d73..23b05d3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,40 @@ +/** + * @packageDocumentation + * + * `@helia/unixfs` is an implementation of a {@link https://github.com/ipfs/specs/blob/main/UNIXFS.md UnixFS filesystem} compatible with {@link https://github.com/ipfs/helia Helia}. + * + * See the {@link UnixFS UnixFS interface} for all available operations. + * + * @example + * + * ```typescript + * import { createHelia } from 'helia' + * import { unixfs } from '@helia/unixfs' + * + * const helia = createHelia({ + * // ... helia config + * }) + * const fs = unixfs(helia) + * + * // create an empty dir and a file, then add the file to the dir + * const emptyDirCid = await fs.addDirectory() + * const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * const updateDirCid = await fs.cp(fileCid, emptyDirCid, 'foo.txt') + * + * // or doing the same thing as a stream + * for await (const entry of fs.addAll([{ + * path: 'foo.txt', + * content: Uint8Array.from([0, 1, 2, 3]) + * }])) { + * console.info(entry) + * } + * ``` + */ + import type { CID, Version } from 'multiformats/cid' import type { Blockstore } from 'interface-blockstore' import type { AbortOptions } from '@libp2p/interfaces' +import { addAll, addBytes, addByteStream, addDirectory, addFile } from './commands/add.js' import { cat } from './commands/cat.js' import { mkdir } from './commands/mkdir.js' import type { Mtime } from 'ipfs-unixfs' @@ -11,50 +45,147 @@ import { touch } from './commands/touch.js' import { chmod } from './commands/chmod.js' import type { UnixFSEntry } from 'ipfs-unixfs-exporter' import { ls } from './commands/ls.js' +import type { ByteStream, DirectoryCandidate, FileCandidate, ImportCandidateStream, ImporterOptions, ImportResult } from 'ipfs-unixfs-importer' export interface UnixFSComponents { blockstore: Blockstore } +/** + * Options to pass to the cat command + */ export interface CatOptions extends AbortOptions { + /** + * Start reading the file at this offset + */ offset?: number + + /** + * Stop reading the file after this many bytes + */ length?: number + + /** + * An optional path to allow reading files inside directories + */ path?: string } +/** + * Options to pass to the chmod command + */ export interface ChmodOptions extends AbortOptions { + /** + * If the target of the operation is a directory and this is true, + * apply the new mode to all directory contents + */ recursive: boolean + + /** + * Optional path to set the mode on directory contents + */ path?: string + + /** + * DAGs with a root block larger than this value will be sharded. Blocks + * smaller than this value will be regular UnixFS directories. + */ shardSplitThresholdBytes: number } +/** + * Options to pass to the cp command + */ export interface CpOptions extends AbortOptions { + /** + * If true, allow overwriting existing directory entries (default: false) + */ force: boolean + + /** + * DAGs with a root block larger than this value will be sharded. Blocks + * smaller than this value will be regular UnixFS directories. + */ shardSplitThresholdBytes: number } +/** + * Options to pass to the ls command + */ export interface LsOptions extends AbortOptions { + /** + * Optional path to list subdirectory contents if the target CID resolves to + * a directory + */ path?: string + + /** + * Start reading the directory entries at this offset + */ offset?: number + + /** + * Stop reading the directory contents after this many directory entries + */ length?: number } +/** + * Options to pass to the mkdir command + */ export interface MkdirOptions extends AbortOptions { + /** + * The CID version to create the new directory with - defaults to the same + * version as the containing directory + */ cidVersion: Version + + /** + * If true, allow overwriting existing directory entries (default: false) + */ force: boolean + + /** + * An optional mode to set on the new directory + */ mode?: number + + /** + * An optional mtime to set on the new directory + */ mtime?: Mtime + + /** + * DAGs with a root block larger than this value will be sharded. Blocks + * smaller than this value will be regular UnixFS directories. + */ shardSplitThresholdBytes: number } +/** + * Options to pass to the rm command + */ export interface RmOptions extends AbortOptions { + /** + * DAGs with a root block larger than this value will be sharded. Blocks + * smaller than this value will be regular UnixFS directories. + */ shardSplitThresholdBytes: number } +/** + * Options to pass to the stat command + */ export interface StatOptions extends AbortOptions { + /** + * An optional path to allow statting paths inside directories + */ path?: string } +/** + * Statistics relating to a UnixFS DAG + */ export interface UnixFSStats { /** * The file or directory CID @@ -109,21 +240,245 @@ export interface UnixFSStats { unixfs?: import('ipfs-unixfs').UnixFS } +/** + * Options to pass to the touch command + */ export interface TouchOptions extends AbortOptions { + /** + * Optional mtime to set on the DAG root, defaults to the current time + */ mtime?: Mtime + + /** + * Optional path to set mtime on directory contents + */ path?: string + + /** + * If the DAG is a directory and this is true, update the mtime on all contents + */ recursive: boolean + + /** + * DAGs with a root block larger than this value will be sharded. Blocks + * smaller than this value will be regular UnixFS directories. + */ shardSplitThresholdBytes: number } +/** + * The UnixFS interface provides familiar filesystem operations to make working with + * UnixFS DAGs simple and intuitive. + */ export interface UnixFS { + /** + * Add all files and directories from the passed stream. This method wraps the + * `importer` export from the `ipfs-unixfs-importer` module - please see the docs + * for input/output types. + * + * @example + * + * ```typescript + * const source = [{ + * path: './foo.txt', + * content: Uint8Array.from([0, 1, 2, 3]) + * }, { + * path: './bar.txt', + * content: Uint8Array.from([4, 5, 6, 7]) + * }] + * + * for await (const entry of fs.import(source)) { + * console.info(entry) + * } + * ``` + */ + addAll: (source: ImportCandidateStream, options?: Partial) => AsyncIterable + + /** + * Add a single `Uint8Array` to your Helia node as a file. + * + * @example + * + * ```typescript + * const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * + * console.info(cid) + * ``` + */ + addBytes: (bytes: Uint8Array, options?: Partial) => Promise + + /** + * Add a stream of `Uint8Array` to your Helia node as a file. + * + * @example + * + * ```typescript + * import fs from 'node:fs' + * + * const stream = fs.createReadStream('./foo.txt') + * const cid = await fs.addByteStream(stream) + * + * console.info(cid) + * ``` + */ + addByteStream: (bytes: ByteStream, options?: Partial) => Promise + + /** + * Add a file to your Helia node with optional metadata. + * + * @example + * + * ```typescript + * const cid = await fs.addFile({ + * path: './foo.txt' + * content: Uint8Array.from([0, 1, 2, 3]), + * mode: 0x755, + * mtime: { + * secs: 10n, + * nsecs: 0 + * } + * }) + * + * console.info(cid) + * ``` + */ + addFile: (file: FileCandidate, options?: Partial) => Promise + + /** + * Add a directory to your Helia node. + * + * @example + * + * ```typescript + * const cid = await fs.addDirectory() + * + * console.info(cid) + * ``` + */ + addDirectory: (dir?: Partial, options?: Partial) => Promise + + /** + * Retrieve the contents of a file from your Helia node. + * + * @example + * + * ```typescript + * for await (const buf of fs.cat(cid)) { + * console.info(buf) + * } + * ``` + */ cat: (cid: CID, options?: Partial) => AsyncIterable - chmod: (source: CID, mode: number, options?: Partial) => Promise + + /** + * Change the permissions on a file or directory in a DAG + * + * @example + * + * ```typescript + * const beforeCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * const beforeStats = await fs.stat(beforeCid) + * + * const afterCid = await fs.chmod(cid, 0x755) + * const afterStats = await fs.stat(afterCid) + * + * console.info(beforeCid, beforeStats) + * console.info(afterCid, afterStats) + * ``` + */ + chmod: (cid: CID, mode: number, options?: Partial) => Promise + + /** + * Add a file or directory to a target directory. + * + * @example + * + * ```typescript + * const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * const directoryCid = await fs.addDirectory() + * + * const updatedCid = await fs.cp(fileCid, directoryCid, 'foo.txt') + * + * console.info(updatedCid) + * ``` + */ cp: (source: CID, target: CID, name: string, options?: Partial) => Promise + + /** + * List directory contents. + * + * @example + * + * ```typescript + * for await (const entry of fs.ls(directoryCid)) { + * console.info(etnry) + * } + * ``` + */ ls: (cid: CID, options?: Partial) => AsyncIterable + + /** + * Make a new directory under an existing directory. + * + * @example + * + * ```typescript + * const directoryCid = await fs.addDirectory() + * + * const updatedCid = await fs.mkdir(directoryCid, 'new-dir') + * + * console.info(updatedCid) + * ``` + */ mkdir: (cid: CID, dirname: string, options?: Partial) => Promise + + /** + * Remove a file or directory from an existing directory. + * + * @example + * + * ```typescript + * const directoryCid = await fs.addDirectory() + * const updatedCid = await fs.mkdir(directoryCid, 'new-dir') + * + * const finalCid = await fs.rm(updatedCid, 'new-dir') + * + * console.info(finalCid) + * ``` + */ rm: (cid: CID, path: string, options?: Partial) => Promise + + /** + * Return statistics about a UnixFS DAG. + * + * @example + * + * ```typescript + * const fileCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * + * const stats = await fs.stat(fileCid) + * + * console.info(stats) + * ``` + */ stat: (cid: CID, options?: Partial) => Promise + + /** + * Update the mtime of a UnixFS DAG + * + * @example + * + * ```typescript + * const beforeCid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) + * const beforeStats = await fs.stat(beforeCid) + * + * const afterCid = await fs.touch(beforeCid) + * const afterStats = await fs.stat(afterCid) + * + * console.info(beforeCid, beforeStats) + * console.info(afterCid, afterStats) + * ``` + */ touch: (cid: CID, options?: Partial) => Promise } @@ -134,12 +489,32 @@ class DefaultUnixFS implements UnixFS { this.components = components } + async * addAll (source: ImportCandidateStream, options: Partial = {}): AsyncIterable { + yield * addAll(source, this.components.blockstore, options) + } + + async addBytes (bytes: Uint8Array, options: Partial = {}): Promise { + return await addBytes(bytes, this.components.blockstore, options) + } + + async addByteStream (bytes: ByteStream, options: Partial = {}): Promise { + return await addByteStream(bytes, this.components.blockstore, options) + } + + async addFile (file: FileCandidate, options: Partial = {}): Promise { + return await addFile(file, this.components.blockstore, options) + } + + async addDirectory (dir: Partial = {}, options: Partial = {}): Promise { + return await addDirectory(dir, this.components.blockstore, options) + } + async * cat (cid: CID, options: Partial = {}): AsyncIterable { yield * cat(cid, this.components.blockstore, options) } - async chmod (source: CID, mode: number, options: Partial = {}): Promise { - return await chmod(source, mode, this.components.blockstore, options) + async chmod (cid: CID, mode: number, options: Partial = {}): Promise { + return await chmod(cid, mode, this.components.blockstore, options) } async cp (source: CID, target: CID, name: string, options: Partial = {}): Promise { @@ -167,6 +542,9 @@ class DefaultUnixFS implements UnixFS { } } +/** + * Create a {@link UnixFS} instance for use with {@link https://github.com/ipfs/helia Helia} + */ export function unixfs (helia: { blockstore: Blockstore }): UnixFS { return new DefaultUnixFS(helia) } diff --git a/test/add.spec.ts b/test/add.spec.ts new file mode 100644 index 0000000..ebc4ba8 --- /dev/null +++ b/test/add.spec.ts @@ -0,0 +1,110 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import type { Blockstore } from 'interface-blockstore' +import { unixfs, UnixFS } from '../src/index.js' +import { MemoryBlockstore } from 'blockstore-core' +import all from 'it-all' + +describe('addAll', () => { + let blockstore: Blockstore + let fs: UnixFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + }) + + it('adds a stream of files', async () => { + const output = await all(fs.addAll([{ + path: './foo.txt', + content: Uint8Array.from([0, 1, 2, 3, 4]) + }, { + path: './bar.txt', + content: Uint8Array.from([5, 4, 3, 2, 1]) + }])) + + expect(output).to.have.lengthOf(2) + expect(output[0].cid.toString()).to.equal('bafkreiaixnpf23vkyecj5xqispjq5ubcwgsntnnurw2bjby7khe4wnjihu') + expect(output[1].cid.toString()).to.equal('bafkreidmuy2n45xj3cdknzprtzo2uvgm3hak6mzy5sllxty457agsftd34') + }) +}) + +describe('addBytes', () => { + let blockstore: Blockstore + let fs: UnixFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + }) + + it('adds bytes', async () => { + const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) + + expect(cid.toString()).to.equal('bafkreiaixnpf23vkyecj5xqispjq5ubcwgsntnnurw2bjby7khe4wnjihu') + }) +}) + +describe('addByteStream', () => { + let blockstore: Blockstore + let fs: UnixFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + }) + + it('adds bytes', async () => { + const cid = await fs.addByteStream([Uint8Array.from([0, 1, 2, 3, 4])]) + + expect(cid.toString()).to.equal('bafkreiaixnpf23vkyecj5xqispjq5ubcwgsntnnurw2bjby7khe4wnjihu') + }) +}) + +describe('addFile', () => { + let blockstore: Blockstore + let fs: UnixFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + }) + + it('adds a file', async () => { + const cid = await fs.addFile({ + content: Uint8Array.from([0, 1, 2, 3, 4]) + }) + + expect(cid.toString()).to.equal('bafkreiaixnpf23vkyecj5xqispjq5ubcwgsntnnurw2bjby7khe4wnjihu') + }) +}) + +describe('addDirectory', () => { + let blockstore: Blockstore + let fs: UnixFS + + beforeEach(async () => { + blockstore = new MemoryBlockstore() + + fs = unixfs({ blockstore }) + }) + + it('adds an empty directory with cidv0', async () => { + const cid = await fs.addDirectory({}, { + cidVersion: 0 + }) + + expect(cid.toString()).to.equal('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + }) + + it('adds an empty directory with no args', async () => { + const cid = await fs.addDirectory() + + expect(cid.toString()).to.equal('bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354') + }) +})