feat: show deal info from w3up (#2573)

Fetch deal information from w3up. There's no way to "multi-get" deal info for more than one upload so I've reworked the deal info in the files list - if it has deals from dagcargo it will just use those, and if it doesn't, it will display a "load deals" button: <img width="1172" alt="Screenshot 2024-04-12 at 12 07 20 PM" src="https://github.com/nftstorage/nft.storage/assets/1113/4799bc48-391d-4d82-9678-93406abfb622"> When clicked, it will find the shards of the upload and find filecoin info for each of them. If it doesn't find any filecoin info in w3up it will display the "queuing" status and tooltip as it did in the past: <img width="1105" alt="Screenshot 2024-04-12 at 12 07 27 PM" src="https://github.com/nftstorage/nft.storage/assets/1113/3151986a-5c96-43b5-b4f7-c9ae2b3a5d3a"> <img width="1155" alt="Screenshot 2024-04-12 at 12 07 34 PM" src="https://github.com/nftstorage/nft.storage/assets/1113/8829c3c0-0a91-4eb4-bb26-a46bda9b82c9"> This required updating the "get NFT" API endpoint to look for filecoin info from w3up. --------- Co-authored-by: Alan Shaw <alan.shaw@protocol.ai>
nftstorage · Apr 15, 2024 · 766a7c1 · 766a7c1
1 parent 6a6298a
commit 766a7c1
Show file tree

Hide file tree

Showing 12 changed files with 616 additions and 173 deletions.
diff --git a/packages/api/package.json b/packages/api/package.json
@@ -33,8 +33,9 @@
     "@ucanto/server": "^9.0.1",
     "@web3-storage/access": "^18.2.0",
     "@web3-storage/car-block-validator": "^1.2.0",
+    "@web3-storage/content-claims": "^4.0.4",
     "@web3-storage/upload-client": "^13.2.0",
-    "@web3-storage/w3up-client": "^12.5.0",
+    "@web3-storage/w3up-client": "^12.5.1",
     "cardex": "^1.0.0",
     "ipfs-car": "^0.6.1",
     "it-last": "^2.0.0",
@@ -47,7 +48,7 @@
     "regexparam": "^2.0.0",
     "toucan-js": "^2.7.0",
     "ucan-storage": "^1.3.0",
-    "uint8arrays": "^3.0.0"
+    "uint8arrays": "5.0.2"
   },
   "devDependencies": {
     "@cloudflare/workers-types": "^3.17.0",
@@ -67,7 +68,7 @@
     "carbites": "^1.0.6",
     "delay": "^5.0.0",
     "dotenv": "^10.0.0",
-    "esbuild": "^0.13.13",
+    "esbuild": "^0.20.2",
     "execa": "^5.1.1",
     "git-rev-sync": "^3.0.1",
     "ipfs-unixfs-importer": "^9.0.3",

diff --git a/packages/api/src/bindings.d.ts b/packages/api/src/bindings.d.ts
@@ -8,6 +8,7 @@ import { DBClient } from './utils/db-client.js'
 import { LinkdexApi } from './utils/linkdex.js'
 import { Logging } from './utils/logs.js'
 import { Client as W3upClient } from '@web3-storage/w3up-client'
+import * as contentClaims from '@web3-storage/content-claims/client'
 
 export type RuntimeEnvironmentName = 'test' | 'dev' | 'staging' | 'production'
 
@@ -142,6 +143,10 @@ export interface AuthOptions {
   checkHasPsaAccess?: boolean
 }
 
+export interface ContentClaimsClient {
+  read: typeof contentClaims.read
+}
+
 export interface RouteContext {
   params: Record<string, string>
   db: DBClient
@@ -158,6 +163,7 @@ export interface RouteContext {
   W3_NFTSTORAGE_SPACE?: string
   W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS?: string
   w3up?: W3upClient
+  contentClaims?: ContentClaimsClient
 }
 
 export type Handler = (

diff --git a/packages/api/src/routes/nfts-get.js b/packages/api/src/routes/nfts-get.js
@@ -3,6 +3,7 @@ import { JSONResponse } from '../utils/json-response.js'
 import { checkAuth, validate } from '../utils/auth.js'
 import { parseCid } from '../utils/utils.js'
 import { toNFTResponse } from '../utils/db-transforms.js'
+import { getW3upDeals } from '../utils/w3up.js'
 
 /**
  * @typedef {import('../bindings').Deal} Deal
@@ -13,8 +14,15 @@ export const nftGet = async (event, ctx) => {
   const { params, db } = ctx
   const { user } = checkAuth(ctx)
   const cid = parseCid(params.cid)
-  const nft = await db.getUpload(cid.sourceCid, user.id)
+  const [nft, w3upDeals] = await Promise.all([
+    db.getUpload(cid.sourceCid, user.id),
+    ctx.w3up && ctx.contentClaims
+      ? getW3upDeals(ctx.w3up, ctx.contentClaims, cid.contentCid)
+      : [],
+  ])
   if (nft) {
+    // merge deals from dagcargo with deals from w3up
+    nft.deals = [...nft?.deals, ...(w3upDeals || [])]
     return new JSONResponse({
       ok: true,
       value: toNFTResponse(nft, cid.sourceCid),

diff --git a/packages/api/src/routes/nfts-upload.js b/packages/api/src/routes/nfts-upload.js
@@ -189,7 +189,6 @@ export async function uploadCarWithStat(
 
     if (stat.structure === 'Partial') {
       checkDagStructureTask = async () => {
-        // @ts-expect-error - I'm not sure why this started failing TODO debug further
         const info = await w3up.capability.upload.get(stat.rootCid)
         if (info.shards && info.shards.length > 1) {
           const structure = await ctx.linkdexApi.getDagStructureForCars(

diff --git a/packages/api/src/utils/context.js b/packages/api/src/utils/context.js
@@ -9,6 +9,7 @@ import { Service } from 'ucan-storage/service'
 import { LinkdexApi } from './linkdex.js'
 import { createW3upClientFromConfig } from './w3up.js'
 import { DID } from '@ucanto/core'
+import * as contentClaims from '@web3-storage/content-claims/client'
 
 /**
  * Obtains a route context object.
@@ -105,6 +106,7 @@ export async function getContext(event, params) {
     r2Uploader,
     log,
     ucanService,
+    contentClaims,
     w3up,
   }
 }
diff --git a/packages/api/src/utils/db-client.js b/packages/api/src/utils/db-client.js
@@ -429,7 +429,7 @@ export class DBClient {
 
     const cids = uploads?.map((u) => u.content_cid)
 
-    const deals = await this.getDealsForCids(cids)
+    const deals = await this.getDealsFromDagcargoFDW(cids)
 
     return uploads?.map((u) => {
       return {
@@ -515,7 +515,7 @@ export class DBClient {
    * @returns {Promise<import('./../bindings').Deal[]>}
    */
   async getDeals(cid) {
-    const deals = await this.getDealsForCids([cid])
+    const deals = await this.getDealsFromDagcargoFDW([cid])
 
     return deals[cid] ? deals[cid] : []
   }
@@ -527,7 +527,7 @@ export class DBClient {
    *
    * @param {string[]} cids
    */
-  async getDealsForCids(cids = []) {
+  async getDealsFromDagcargoFDW(cids = []) {
     try {
       const rsp = await this.client.rpc('find_deals_by_content_cids', {
         cids,

diff --git a/packages/api/src/utils/w3up.js b/packages/api/src/utils/w3up.js
@@ -7,6 +7,7 @@ import { identity } from 'multiformats/hashes/identity'
 import { CarReader } from '@ipld/car'
 import { importDAG } from '@ucanto/core/delegation'
 import * as W3upClient from '@web3-storage/w3up-client'
+import { parseLink } from '@ucanto/core'
 import { connect } from '@ucanto/client'
 import { CAR, HTTP } from '@ucanto/transport'
 
@@ -106,3 +107,95 @@ export async function createW3upClientFromConfig(options) {
   await w3up.addSpace(await parseW3Proof(options.proof))
   return w3up
 }
+
+/**
+ *
+ * @param {W3upClient.Client} client
+ * @param {{read: typeof import('@web3-storage/content-claims/client').read}} contentClaimsClient
+ * @param {import('@web3-storage/upload-client/types').UploadListItem} upload
+ * @returns {Promise<import('@web3-storage/access').Result<import('@web3-storage/access').FilecoinInfoSuccess>[]>}
+ */
+async function getFilecoinInfos(client, contentClaimsClient, upload) {
+  return await Promise.all(
+    // for each shard of the upload
+    upload.shards
+      ? upload.shards.map(async (shard) => {
+          // find the equivalent piece link
+          const pieceClaims = await contentClaimsClient.read(shard)
+          const pieceClaim =
+            /** @type {import('@web3-storage/content-claims/client/api').EqualsClaim} */ (
+              pieceClaims.find((c) => c.type === 'assert/equals')
+            )
+          if (pieceClaim) {
+            const pieceLink = pieceClaim.equals
+            // and get filecoin info for it
+            const filecoinInfo = await client.capability.filecoin.info(
+              /** @type {import('@web3-storage/access').PieceLink} */ (
+                pieceLink
+              )
+            )
+            return filecoinInfo.out
+          } else {
+            return {
+              error: {
+                name: 'PieceLinkClaimNotFound',
+                message: `could not find piece link equivalent of ${shard}`,
+              },
+            }
+          }
+        })
+      : []
+  )
+}
+
+/**
+ *
+ * @param {W3upClient.Client | undefined} client
+ * @param {{read: typeof import('@web3-storage/content-claims/client').read}} contentClaimsClient
+ * @param {string} contentCid
+ * @returns {Promise<import('../bindings').Deal[]>}
+ */
+export async function getW3upDeals(client, contentClaimsClient, contentCid) {
+  if (client) {
+    const link = parseLink(contentCid)
+    // get the upload
+    let upload
+    try {
+      upload = await client.capability.upload.get(link)
+    } catch (e) {
+      console.error('error getting upload', e)
+      return []
+    }
+    const filecoinInfoResults = await getFilecoinInfos(
+      client,
+      contentClaimsClient,
+      upload
+    )
+    /**
+     * @type {import('../bindings').Deal[]}
+     */
+    const filecoinInfos = []
+    for (const result of filecoinInfoResults) {
+      if (result.ok) {
+        const info = result.ok
+        for (const deal of info.deals) {
+          filecoinInfos.push({
+            pieceCid: info.piece.toString(),
+            status: 'published',
+            batchRootCid: deal.aggregate.toString(),
+            miner: deal.provider,
+            chainDealID: Number(deal.aux.dataSource.dealID),
+            // TODO: figure this out
+            datamodelSelector: '',
+          })
+        }
+      } else {
+        // @ts-expect-error - in practice this will just be undefined if message doesn't exist
+        console.warn(`error getting filecoininfo: ${result.error.message}`)
+      }
+    }
+    return filecoinInfos
+  } else {
+    return []
+  }
+}
diff --git a/packages/api/test/nfts-get.spec.js b/packages/api/test/nfts-get.spec.js
@@ -1,13 +1,146 @@
 import test from 'ava'
+import { createServer } from 'node:http'
+import { ed25519 } from '@ucanto/principal'
+import { delegate, parseLink } from '@ucanto/core'
+import { base64 } from 'multiformats/bases/base64'
 import { createClientWithUser } from './scripts/helpers.js'
 import { fixtures } from './scripts/fixtures.js'
 import {
   getMiniflareContext,
   setupMiniflareContext,
 } from './scripts/test-context.js'
+import {
+  createMockW3up,
+  locate,
+  encodeDelegationAsCid,
+} from './utils/w3up-testing.js'
+
+const nftStorageSpace = ed25519.generate()
+const nftStorageApiPrincipal = ed25519.generate()
+const nftStorageAccountEmailAllowListedForW3up = 'test+w3up@dev.nft.storage'
+const mockW3upDID = 'did:web:test.web3.storage'
+/**
+ * @type {import('@web3-storage/access').PieceLink}
+ */
+const mockPieceLink = parseLink(
+  'bafkzcibeslzwmewd4pugjanyiayot5m76a67dvdir25v6ms6kbuozy2sxotplrrrce'
+)
+/**
+ * @type {import('@web3-storage/access').FilecoinInfoAcceptedDeal[]}
+ */
+const mockDeals = [
+  {
+    aggregate: parseLink(
+      'bafkzcibcaapen7lfjgljzi523a5rau2l5pwpwseita6uunqy5otrlxa2l2pouca'
+    ),
+    aux: {
+      dataSource: {
+        dealID: BigInt(1),
+      },
+      dataType: BigInt(1),
+    },
+    provider: 'f01240',
+  },
+]
+const cidWithShards = parseLink(
+  'bafybeiccy35oi3gajocq5bbg7pnaxb3kv5ibtdz3tc3kari53qhbjotzey'
+)
+const mockW3up = Promise.resolve(
+  (async function () {
+    const server = createServer(
+      await createMockW3up({
+        did: mockW3upDID,
+        // @ts-expect-error not returning a full upload get response for now
+        async onHandleUploadGet(invocation) {
+          if (invocation.capability.nb.root?.equals(cidWithShards)) {
+            return {
+              // grabbed this shard CID from staging, it should correspond to a piece named bafkzcibeslzwmewd4pugjanyiayot5m76a67dvdir25v6ms6kbuozy2sxotplrrrce
+              shards: [
+                parseLink(
+                  'bagbaieragf62xatg3bqrfafdy3lpk2fte7526kvxnltqsnhjr45cz6jjk7mq'
+                ),
+              ],
+            }
+          } else {
+            return {
+              shards: [],
+            }
+          }
+        },
+        async onHandleFilecoinInfo(invocation) {
+          if (invocation.capability.nb.piece.equals(mockPieceLink)) {
+            return {
+              deals: mockDeals,
+              aggregates: [],
+              piece: mockPieceLink,
+            }
+          } else {
+            return undefined
+          }
+        },
+      })
+    )
+    server.listen(0)
+    await new Promise((resolve) =>
+      server.addListener('listening', () => resolve(undefined))
+    )
+    return {
+      server,
+    }
+  })()
+)
 
 test.before(async (t) => {
-  await setupMiniflareContext(t)
+  await setupMiniflareContext(t, {
+    overrides: {
+      W3UP_URL: locate((await mockW3up).server).url.toString(),
+      W3UP_DID: mockW3upDID,
+      W3_NFTSTORAGE_SPACE: (await nftStorageSpace).did(),
+      W3_NFTSTORAGE_PRINCIPAL: ed25519.format(await nftStorageApiPrincipal),
+      W3_NFTSTORAGE_PROOF: (
+        await encodeDelegationAsCid(
+          await delegate({
+            issuer: await nftStorageSpace,
+            audience: await nftStorageApiPrincipal,
+            capabilities: [
+              { can: 'upload/get', with: (await nftStorageSpace).did() },
+              { can: 'filecoin/info', with: (await nftStorageSpace).did() },
+            ],
+          })
+        )
+      ).toString(base64),
+      W3_NFTSTORAGE_ENABLE_W3UP_FOR_EMAILS: JSON.stringify([
+        nftStorageAccountEmailAllowListedForW3up,
+      ]),
+    },
+  })
+})
+
+test.serial('should fetch deal details from w3up', async (t) => {
+  const cid = cidWithShards.toString()
+  const client = await createClientWithUser(t)
+  const mf = getMiniflareContext(t)
+  await client.addPin({
+    cid,
+    name: 'test-filecoin-info',
+  })
+
+  const res = await mf.dispatchFetch(`http://miniflare.test/${cid}`, {
+    headers: { Authorization: `Bearer ${client.token}` },
+  })
+  const { ok, value } = await res.json()
+  t.assert(ok)
+  t.deepEqual(
+    value.deals,
+    mockDeals.map((deal) => ({
+      pieceCid: mockPieceLink.toString(),
+      status: 'published',
+      datamodelSelector: '',
+      batchRootCid: deal.aggregate.toString(),
+      miner: deal.provider,
+      chainDealID: Number(deal.aux.dataSource.dealID),
+    }))
+  )
 })
 
 test.serial('should return proper response for cid v1', async (t) => {