feat: add file

web3-storage · Jul 1, 2021 · 46f7b98 · 46f7b98
1 parent 3c1b665
commit 46f7b98
Show file tree

Hide file tree

Showing 20 changed files with 1,180 additions and 759 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/client/examples/browser/.gitignore b/packages/client/examples/browser/.gitignore
@@ -0,0 +1,5 @@
+node_modules
+.DS_Store
+dist
+dist-ssr
+*.local
diff --git a/packages/client/examples/browser/README.md b/packages/client/examples/browser/README.md
@@ -0,0 +1,18 @@
+# Files demo browser - web3.storage
+
+🚧  **WORK IN PROGRESS**  🚧
+
+A demo using web3.storage client in the browser to pre-calculate the CID for an asset then storing it on tbd.storage and confirming that it uses the exact same CID for the asset.
+
+## Getting started
+
+```console
+npm install
+npm run dev
+
+# or
+yarn
+yarn dev
+```
+
+Then visit `http://localhost:3000?key=<your web3.storage API KEY here>`
diff --git a/packages/client/examples/browser/index.html b/packages/client/examples/browser/index.html
@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>CAR upload - nft.storage</title>
+  </head>
+  <body>
+    <pre id="out"></pre>
+    <script type="module" src="/main.js"></script>
+  </body>
+</html>
diff --git a/packages/client/examples/browser/main.js b/packages/client/examples/browser/main.js
@@ -0,0 +1,51 @@
+import { Web3Storage } from 'web3.storage'
+import { Web3File } from 'web3-file'
+
+const endpoint = 'https://api.web3.storage' // the default
+const token =
+  new URLSearchParams(window.location.search).get('key') || 'API_KEY' // your API key from https://web3.storage/manage
+
+async function main() {
+  const storage = new Web3Storage({ endpoint, token })
+
+  const files = prepareFiles()
+
+  // send the files to tbd.storage
+  const cid = await storage.put(files)
+
+  // TODO
+  console.log('added', cid)
+  // check that the CID is pinned
+  // const status = await store.status(cid)
+  // log(status)
+}
+
+function prepareFiles () {
+  const data = 'Hello web3.storage!'
+  const data2 = 'Hello web3.storage!!'
+
+  return [
+    Web3File.fromBytes(
+      new TextEncoder().encode(data),
+      'data.zip',
+      { path: '/dir/data.zip' }
+    ),
+    Web3File.fromBytes(
+      new TextEncoder().encode(data2),
+      'data2.zip',
+      { path: '/dir/data2.zip' }
+    ),
+    Web3File.fromBytes(
+      new TextEncoder().encode(data),
+      'data.zip',
+      { path: '/dir/dir/data.zip' }
+    ),
+    Web3File.fromBytes(
+      new TextEncoder().encode(data2),
+      'data2.zip',
+      { path: '/dir/dir/data2.zip' }
+    )
+  ]
+}
+
+main()
diff --git a/packages/client/examples/browser/package.json b/packages/client/examples/browser/package.json
@@ -0,0 +1,14 @@
+{
+  "version": "0.0.0",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "serve": "vite preview"
+  },
+  "devDependencies": {
+    "vite": "^2.3.7"
+  },
+  "dependencies": {
+    "web3.storage": "../../"
+  }
+}
diff --git a/packages/client/examples/node.js/README.md b/packages/client/examples/node.js/README.md
@@ -0,0 +1,3 @@
+# Files demo Node.js - web3.storage
+
+🚧  **WORK IN PROGRESS**  🚧
diff --git a/packages/client/examples/node.js/files.js b/packages/client/examples/node.js/files.js
@@ -0,0 +1,46 @@
+import { Web3Storage } from '../../src/lib.js'
+import { Web3File } from 'web3-file'
+
+// TODO
+const endpoint = 'https://api.web3.storage' // the default
+const token = 'API_KEY' // your API key from https://web3.storage/manage
+
+async function main() {
+  const storage = new Web3Storage({ endpoint, token }) 
+
+  const files = prepareFiles()
+  const cid = await storage.put(files)
+
+  console.log('added', cid)
+}
+
+// TODO: Read a fixstures folder instead
+function prepareFiles () {
+  const data = 'Hello web3.storage!'
+  const data2 = 'Hello web3.storage!!'
+
+  return [
+    Web3File.fromBytes(
+      new TextEncoder().encode(data),
+      'data.zip',
+      { path: '/dir/data.zip' }
+    ),
+    Web3File.fromBytes(
+      new TextEncoder().encode(data2),
+      'data2.zip',
+      { path: '/dir/data2.zip' }
+    ),
+    Web3File.fromBytes(
+      new TextEncoder().encode(data),
+      'data.zip',
+      { path: '/dir/dir/data.zip' }
+    ),
+    Web3File.fromBytes(
+      new TextEncoder().encode(data2),
+      'data2.zip',
+      { path: '/dir/dir/data2.zip' }
+    )
+  ]
+}
+
+main()
diff --git a/packages/client/examples/node.js/package.json b/packages/client/examples/node.js/package.json
@@ -0,0 +1,12 @@
+{
+  "name": "filecoin-storage-examples",
+  "version": "0.0.0",
+  "private": true,
+  "description": "Examples of using filecoin.storage in Node.js",
+  "type": "module",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Vasco Santos",
+  "license": "(Apache-2.0 AND MIT)"
+}
diff --git a/packages/client/package.json b/packages/client/package.json
@@ -32,8 +32,13 @@
     "@ipld/car": "^3.1.2",
     "@web-std/blob": "^2.1.0",
     "@web-std/fetch": "^2.0.1",
+    "@web-std/file": "^1.1.0",
     "browser-readablestream-to-it": "^1.0.2",
-    "ipfs-car": "^0.3.5"
+    "carbites": "^1.0.6",
+    "ipfs-car": "^0.3.5",
+    "p-retry": "^4.5.0",
+    "streaming-iterables": "^6.0.0",
+    "web3-file": "^0.1.0"
   },
   "devDependencies": {
     "@rollup/plugin-commonjs": "^19.0.0",

diff --git a/packages/client/src/lib.js b/packages/client/src/lib.js
@@ -6,40 +6,52 @@
  *
  * @example
  * ```js
- * import { FilecoinStorage, Blob } from "web3.storage"
- * const client = new FilecoinStorage({ token: API_TOKEN })
+ * import { Web3Storage, Blob } from "web3.storage"
+ * const client = new Web3Storage({ token: API_TOKEN })
  *
  * const cid = await client.storeBlob(new Blob(['hello world']))
  * ```
  * @module
  */
+import { transform } from 'streaming-iterables'
+import pRetry from 'p-retry'
+import { pack } from 'ipfs-car/pack'
+import { TreewalkCarSplitter } from 'carbites/treewalk'
 import * as API from './lib/interface.js'
-import { fetch, Blob } from './platform.js'
+import {
+  fetch,
+  Blob,
+  Blockstore
+} from './platform.js'
 import { CarReader } from '@ipld/car/reader'
 import { unpack } from 'ipfs-car/unpack'
 import toIterable from 'browser-readablestream-to-it'
 
+const MAX_ADD_RETRIES = 5
+const MAX_CONCURRENT_UPLOADS = 3
+const MAX_CHUNK_SIZE = 1024 * 1024 * 10 // chunk to ~10MB CARs
+
 /**
  * @implements API.Service
  */
-class FilecoinStorage {
+class Web3Storage {
   /**
    * Constructs a client bound to the given `options.token` and
    * `options.endpoint`.
    *
    * @example
    * ```js
-   * import { FilecoinStorage, Blob } from "web3.storage"
-   * const client = new FilecoinStorage({ token: API_TOKEN })
+   * import { Web3Storage, Blob } from "web3.storage"
+   * const client = new Web3Storage({ token: API_TOKEN })
    * const { car, rootCid } = await client.pack(new Blob(['hello world']))
    * const cid = await client.store(car)
    * console.assert(cid === rootCid, 'The service should store the files with the `rootCid` I created')
    * ```
    * Optionally you could pass an alternative API endpoint (e.g. for testing)
    * @example
    * ```js
-   * import { FilecoinStorage } from "web3.storage"
-   * const client = new FilecoinStorage({
+   * import { Web3Storage } from "web3.storage"
+   * const client = new Web3Storage({
    *   token: API_TOKEN
    *   endpoint: new URL('http://localhost:8080/')
    * })
@@ -75,33 +87,66 @@ class FilecoinStorage {
 
   /**
    * @param {API.Service} service
-   * @param {Blob} blob
+   * @param {Iterable<API.Web3File>} files
+   * @param {{onStoredChunk?: (size: number) => void}} [options]
    * @returns {Promise<API.CIDString>}
    */
-  static async store({ endpoint, token }, blob) {
+  static async put({ endpoint, token }, files, { onStoredChunk } = {}) {
     const url = new URL(`/car`, endpoint)
+    const headers = Web3Storage.headers(token)
+    const targetSize = MAX_CHUNK_SIZE
 
-    if (blob.size === 0) {
-      throw new Error('Content size is 0, make sure to provide some content')
-    }
+    const blockstore = new Blockstore()
+    const { out } = await pack({
+      input: files,
+      blockstore
+    })
+    const splitter = await TreewalkCarSplitter.fromIterable(out, targetSize)
 
-    const car =
-      blob.type !== 'application/car'
-        ? blob.slice(0, blob.size, 'application/car')
-        : blob
+    const upload = transform(
+      MAX_CONCURRENT_UPLOADS,
+      async (/** @type {AsyncIterable<Uint8Array>} */ car) => {
+        const carParts = []
+        for await (const part of car) {
+          carParts.push(part)
+        }
 
-    const request = await fetch(url.toString(), {
-      method: 'POST',
-      headers: FilecoinStorage.headers(token),
-      body: car,
-    })
-    const result = await request.json()
+        const carFile = new Blob(carParts, {
+          type: 'application/car',
+        })
+
+        const res = await pRetry(
+          async () => {
+            const request = await fetch(url.toString(), {
+              method: 'POST',
+              headers,
+              body: carFile,
+            })
+            const result = await request.json()
 
-    if (result.ok) {
-      return result.value.cid
-    } else {
-      throw new Error(result.error.message)
+            if (result.ok) {
+              return result.value.cid
+            } else {
+              throw new Error(result.error.message)
+            }
+          },
+          { retries: MAX_ADD_RETRIES }
+        )
+        onStoredChunk && onStoredChunk(carFile.size)
+        return res
+      }
+    )
+
+    let root
+    for await (const cid of upload(splitter.cars())) {
+      root = cid
     }
+
+    // Destroy Blockstore
+    await blockstore.destroy()
+
+    // @ts-ignore there will always be a root, or carbites will fail
+    return root
   }
 
   /**
@@ -113,7 +158,7 @@ class FilecoinStorage {
     const url = new URL(`/car/${cid}`, endpoint)
     const res = await fetch(url.toString(), {
       method: 'GET',
-      headers: FilecoinStorage.headers(token),
+      headers: Web3Storage.headers(token),
     })
     if (!res.ok) {
       // TODO: I'm assuming that an error for "CID isn't there (yet)" would be unergonomic. Need to verify.
@@ -130,9 +175,7 @@ class FilecoinStorage {
   // Just a sugar so you don't have to pass around endpoint and token around.
 
   /**
-   * Stores files encoded as a single [Content Addressed Archive
-   * (CAR)](https://github.com/ipld/specs/blob/master/block-layer/content-addressable-archives.md).
-   *
+   * Puts files.
    * Takes a [Blob](https://developer.mozilla.org/en-US/docs/Web/API/Blob/Blob)
    *
    * Returns the corresponding Content Identifier (CID).
@@ -144,18 +187,19 @@ class FilecoinStorage {
    * const cid = await client.store(car)
    * console.assert(cid === root)
    * ```
-   * @param {Blob} blob
+   * @param {Iterable<API.Web3File>} files
+   * @param {{onStoredChunk?: (size: number) => void}} [options]
    */
-  store(blob) {
-    return FilecoinStorage.store(this, blob)
+  put(files, options) {
+    return Web3Storage.put(this, files, options)
   }
 
   /**
    * Fetch the Content Addressed Archive by it's root CID
    * @param {string} cid
    */
   get(cid) {
-    return FilecoinStorage.get(this, cid)
+    return Web3Storage.get(this, cid)
   }
 }
 
@@ -232,11 +276,11 @@ function toCarResponse(res) {
   return response
 }
 
-export { FilecoinStorage, Blob }
+export { Web3Storage, Blob }
 
 /**
  * Just to verify API compatibility.
  * @type {API.API}
  */
-const api = FilecoinStorage
+const api = Web3Storage
 void api