feat: client.get: (cid) => Responsish (#4)

- Have client.get return an object with `files` and `filesIterator` methods on it.
- Add rollup config to make a single file esm bundle
- Add bundlesize so we can see the impact of adding CAR support
- Build and test the client in CI.


License: (Apache-2.0 AND MIT)
Signed-off-by: Oli Evans <oli@tableflip.io>

.github/workflows/client.yml

@@ -0,0 +1,20 @@
+name: Client
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    name: Test
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 16
+      - uses: bahmutov/npm-install@v1
+      - run: npm run build --workspace packages/client
+      - run: npm test --workspace packages/client

package.json

@@ -8,7 +8,7 @@
     "packages/client"
   ],
   "scripts": {
-    "test": "say sorry",
+    "test": "npm run test --workspaces",
     "format": "prettier --write '**/*.{js,jsx,ts,tsx}'"
   },
   "devDependencies": {

packages/client/package.json

@@ -14,34 +14,41 @@
   "homepage": "https://github.com/filecoin-storage/filecoin.storage#readme",
   "scripts": {
     "test": "npm-run-all -p -r mock:api test:all",
-    "test:all": "run-s test:web test:cjs test:es",
+    "test:all": "run-s test:web test:cjs test:esm test:size",
+    "test:size": "bundlesize",
     "test:web": "API_PORT=1337 playwright-test test/*.spec.js",
     "test:cjs": "API_PORT=1337 mocha dist/test/*.spec.cjs --exit",
-    "test:es": "API_PORT=1337 hundreds mocha test/*.spec.js --exit",
+    "test:esm": "API_PORT=1337 hundreds mocha test/*.spec.js --exit",
     "mock:api": "smoke -p 1337 --hooks test/mocks/hooks.js test/mocks/api",
     "build": "run-p build:*",
-    "build:ts": "tsc --build",
     "build:cjs": "rollup --config --silent rollup.config.js",
+    "build:esm": "rollup --config rollup.esm.config.js",
+    "build:tsc": "tsc --build",
     "coverage": "nyc report --reporter=text-lcov > coverage.lcov && npx codecov",
     "typedoc": "typedoc --entryPoints src  --out ../../docs/client",
     "prepare": "npm run build"
   },
   "dependencies": {
-    "@ipld/car": "^2.1.0",
+    "@ipld/car": "^3.1.2",
     "@web-std/blob": "^2.1.0",
-    "@web-std/fetch": "^2.0.1"
+    "@web-std/fetch": "^2.0.1",
+    "browser-readablestream-to-it": "^1.0.2",
+    "ipfs-car": "^0.3.5"
   },
   "devDependencies": {
+    "@rollup/plugin-commonjs": "^19.0.0",
+    "@rollup/plugin-node-resolve": "^13.0.0",
     "@types/mocha": "8.2.2",
+    "bundlesize": "^0.18.1",
     "hundreds": "0.0.9",
-    "ipfs-car": "^0.2.4",
     "mocha": "8.3.2",
     "multiformats": "^7.0.0",
     "npm-run-all": "^4.1.5",
     "nyc": "15.1.0",
     "playwright-test": "^4.1.0",
     "rollup": "2.50.1",
     "rollup-plugin-multi-input": "1.3.1",
+    "rollup-plugin-terser": "^7.0.2",
     "smoke": "^3.1.1",
     "typedoc": "0.20.36",
     "uvu": "0.5.1"
@@ -63,5 +70,11 @@
   },
   "browser": {
     "./src/platform.js": "./src/platform.web.js"
-  }
+  },
+  "bundlesize": [
+    {
+      "path": "./dist/bundle.esm.min.js",
+      "maxSize": "200 kB"
+    }
+  ]
 }

packages/client/rollup.esm.config.js

@@ -0,0 +1,22 @@
+// @ts-ignore
+import { terser } from 'rollup-plugin-terser'
+import commonjs from '@rollup/plugin-commonjs'
+import resolve from '@rollup/plugin-node-resolve'
+
+export default {
+  input: 'src/lib.js',
+  output: [
+    {
+      file: 'dist/bundle.esm.min.js',
+      format: 'esm',
+      plugins: [terser()],
+      sourcemap: true,
+    },
+  ],
+  plugins: [
+    commonjs(),
+    resolve({
+      browser: true,
+    }),
+  ],
+}

packages/client/src/lib.js

@@ -15,6 +15,9 @@
  */
 import * as API from './lib/interface.js'
 import { fetch, Blob } from './platform.js'
+import { CarReader } from '@ipld/car/reader'
+import { unpack } from 'ipfs-car/unpack'
+import toIterable from 'browser-readablestream-to-it'
 
 /**
  * @implements API.Service
@@ -104,6 +107,7 @@ class FilecoinStorage {
   /**
    * @param {API.Service} service
    * @param {string} cid
+   * @returns {Promise<import('./lib/interface.js').CarResponse | null>}
    */
   static async get({ endpoint, token }, cid) {
     const url = new URL(`/car/${cid}`, endpoint)
@@ -120,7 +124,7 @@ class FilecoinStorage {
         throw new Error(`${res.status} ${res.statusText}`)
       }
     }
-    return res.blob()
+    return toCarResponse(res)
   }
 
   // Just a sugar so you don't have to pass around endpoint and token around.
@@ -155,6 +159,79 @@ class FilecoinStorage {
   }
 }
 
+/**
+ * Upgrade a ReadableStream to an AsyncIterable if it isn't already
+ *
+ * ReadableStream (e.g res.body) is asyncIterable in node, but not in chrome, yet.
+ * see: https://bugs.chromium.org/p/chromium/issues/detail?id=929585
+ *
+ * @param {ReadableStream<Uint8Array>} readable
+ * @returns {AsyncIterable<Uint8Array>}
+ */
+function asAsyncIterable(readable) {
+  // @ts-ignore how to tell tsc that we are checking the type here?
+  return Symbol.asyncIterator in readable
+    ? readable
+    : /* c8 ignore next */
+      toIterable(readable)
+}
+
+/**
+ * map a UnixFSEntry to a ~Blob~ File with benefits
+ * @param {import('./lib/interface.js').UnixFSEntry} e
+ * @returns {Promise<import('./lib/interface.js').IpfsFile>}
+ */
+async function toIpfsFile(e) {
+  const chunks = []
+  for await (const chunk of e.content()) {
+    chunks.push(chunk)
+  }
+
+  // A Blob in File clothing
+  const file = Object.assign(new Blob(chunks), {
+    cid: e.cid.toString(),
+    name: e.name,
+    relativePath: e.path,
+    webkitRelativePath: e.path,
+    // TODO: mtime may be available on UnixFSEntry... need to investigate ts weirdness.
+    lastModified: Date.now(),
+  })
+  return file
+}
+
+/**
+ * Add car unpacking smarts to the response object,
+ * @param {Response} res
+ * @returns {import('./lib/interface.js').CarResponse}
+ */
+function toCarResponse(res) {
+  const response = Object.assign(res, {
+    unixFsIterator: async function* () {
+      /* c8 ignore next 3 */
+      if (!res.body) {
+        throw new Error('No body on response')
+      }
+      const carReader = await CarReader.fromIterable(asAsyncIterable(res.body))
+      for await (const entry of unpack(carReader)) {
+        yield entry
+      }
+    },
+    files: async () => {
+      const files = []
+      // @ts-ignore we're using the enriched response here
+      for await (const entry of response.unixFsIterator()) {
+        if (entry.type === 'directory') {
+          continue
+        }
+        const file = await toIpfsFile(entry)
+        files.push(file)
+      }
+      return files
+    },
+  })
+  return response
+}
+
 export { FilecoinStorage, Blob }
 
 /**

packages/client/src/lib/interface.ts

@@ -1,6 +1,6 @@
+import type { UnixFSEntry } from 'ipfs-car/unpack'
 import type { CID } from 'multiformats'
-
-export type { CID }
+export type { CID , UnixFSEntry }
 
 /**
  * Define nominal type of U based on type of T. Similar to Opaque types in Flow
@@ -29,97 +29,14 @@ export interface API {
   /**
    * Get files for a root CID packed as a CAR file
    */
-  get(service: Service, cid: CIDString): Promise<Blob | null>
+  get(service: Service, cid: CIDString): Promise<CarResponse | null>
 }
 
-export interface StatusResult {
-  cid: string
-  size: number
-  deals: Deal[]
-  pin: Pin
-  created: Date
+export interface IpfsFile extends File {
+  cid: CIDString,  
 }
 
-export type Deal =
-  | QueuedDeal
-  | PendingDeal
-  | FailedDeal
-  | PublishedDeal
-  | FinalizedDeal
-
-export interface DealInfo {
-  lastChanged: Date
-  /**
-   * Miner ID
-   */
-  miner: string
-
-  /**
-   * Filecoin network for this Deal
-   */
-  network?: 'nerpanet' | 'mainnet'
-  /**
-   * Piece CID string
-   */
-  pieceCid: CIDString
-  /**
-   * CID string
-   */
-  batchRootCid: CIDString
-}
-
-export interface QueuedDeal {
-  status: 'queued'
-  /**
-   * Timestamp in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format: YYYY-MM-DDTHH:MM:SSZ.
-   */
-  lastChanged: Date
+export interface CarResponse extends Response {
+  unixFsIterator: () => AsyncIterable<UnixFSEntry>
+  files: () => Promise<Array<IpfsFile>>
 }
-
-export interface PendingDeal extends DealInfo {
-  status: 'proposing' | 'accepted'
-}
-
-export interface FailedDeal extends DealInfo {
-  status: 'failed'
-  /**
-   * Reason deal failed.
-   */
-  statusText: string
-}
-
-export interface PublishedDeal extends DealInfo {
-  status: 'published'
-  /**
-   * Identifier for the deal stored on chain.
-   */
-  chainDealID: number
-}
-
-export interface FinalizedDeal extends DealInfo {
-  status: 'active' | 'terminated'
-  /**
-   * Identifier for the deal stored on chain.
-   */
-  chainDealID: number
-
-  /**
-   * Deal Activation
-   */
-  dealActivation: Date
-  /**
-   * Deal Expiraction
-   */
-  dealExpiration: Date
-}
-
-export interface Pin {
-  // Pinata does not provide this
-  // requestid: string
-  cid: CIDString
-  name?: string
-  status: PinStatus
-  created: Date
-}
-
-export type PinStatus = 'queued' | 'pinning' | 'pinned' | 'failed'

packages/client/src/platform.js

@@ -1,10 +1,4 @@
 import fetch, { Request, Response, Headers } from '@web-std/fetch'
 import { Blob } from '@web-std/blob'
 
-export {
-  fetch,
-  Request,
-  Response,
-  Headers,
-  Blob
-}
+export { fetch, Request, Response, Headers, Blob }

packages/client/src/platform.web.js

@@ -1,2 +1,5 @@
 export const fetch = globalThis.fetch
+export const Request = globalThis.Request
+export const Response = globalThis.Response
 export const Blob = globalThis.Blob
+export const File = globalThis.File