feat: listUploads includes parts property with list of links to CARs …

…that contain it (#2347)

Motivation:
* #2343

Why touch the sql files?
* the db tests wouldn't run cleanly without these changes. It would try
to drop tables but fail due to dependencies on the uuid extension, other
cols, etc. I think these changes are safe/good and will help others too
but I can remove them if needed

---------

Co-authored-by: Alan Shaw <alan.shaw@protocol.ai>

.github/workflows/w3.yml

@@ -43,6 +43,9 @@ jobs:
       # We want this one to fail if we can't upload to the staging api using the workspace version of the client.
       - uses: bahmutov/npm-install@v1
       - name: Test upload to staging
+        # disabled until we can make this succeed while staging is in maintenance mode
+        # as part of old.web3.storage sunset
+        continue-on-error: true
         run: |
           npm run build -w packages/client
           echo "$(date --utc --iso-8601=seconds) web3.storage upload test" > ./upload-test-small

packages/api/src/magic.link.js

@@ -58,7 +58,12 @@ function createMagicTestmodeBypasss () {
 export const magicLinkBypassForE2ETestingInTestmode = createMagicTestmodeBypasss()
 
 function isMagicTestModeToken (token) {
-  const parsed = JSON.parse(globalThis.atob(token))
+  let parsed
+  try {
+    parsed = JSON.parse(globalThis.atob(token))
+  } catch {
+    return false
+  }
   if (parsed.length !== 2) {
     // unexpeced parse
     return false

packages/api/test/fixtures/pgrest/get-user-uploads.js

@@ -29,7 +29,8 @@ export default [
       }
     ],
     type: 'Car',
-    updated: '2021-07-14T19:27:14.934572+00:00'
+    updated: '2021-07-14T19:27:14.934572+00:00',
+    parts: []
   },
   {
     _id: '8',
@@ -48,7 +49,8 @@ export default [
       }
     ],
     type: 'Car',
-    updated: '2021-07-14T19:27:14.934572+00:00'
+    updated: '2021-07-14T19:27:14.934572+00:00',
+    parts: []
   },
   {
     _id: '1',
@@ -59,7 +61,8 @@ export default [
     created: '2021-07-09T16:20:33.946845+00:00',
     updated: '2021-07-09T16:20:33.946845+00:00',
     deals: [],
-    pins: []
+    pins: [],
+    parts: []
   },
   {
     _id: '2',
@@ -70,7 +73,8 @@ export default [
     created: '2021-07-09T10:40:35.408884+00:00',
     updated: '2021-07-09T10:40:35.408884+00:00',
     deals: [],
-    pins: []
+    pins: [],
+    parts: []
   },
   {
     _id: '3',
@@ -81,6 +85,7 @@ export default [
     created: '2021-07-09T10:36:05.862862+00:00',
     updated: '2021-07-09T10:36:05.862862+00:00',
     deals: [],
-    pins: []
+    pins: [],
+    parts: []
   }
 ]

packages/db/db-client-types.ts

@@ -206,6 +206,7 @@ export type UploadItem = {
   created?: definitions['upload']['inserted_at']
   updated?: definitions['upload']['updated_at']
   content: ContentItem
+  backupUrls: definitions['upload']['backup_urls']
 }
 
 export type UploadItemOutput = {
@@ -218,6 +219,11 @@ export type UploadItemOutput = {
   dagSize?: definitions['content']['dag_size']
   pins: Array<PinItemOutput>,
   deals: Array<Deal>
+  /**
+   * the graph from `cid` can be recreated from the blocks in these parts
+   * @see https://github.com/web3-storage/content-claims#partition-claim
+   */
+  parts: Array<string>
 }
 
 export type UploadOutput = definitions['upload'] & {

packages/db/index.js

@@ -21,6 +21,7 @@ const uploadQuery = `
         sourceCid:source_cid,
         created:inserted_at,
         updated:updated_at,
+        backupUrls:backup_urls,
         content(cid, dagSize:dag_size, pins:pin(status, updated:updated_at, location:pin_location(_id:id, peerId:peer_id, peerName:peer_name, ipfsPeerId:ipfs_peer_id, region)))
       `
 
@@ -555,7 +556,6 @@ export class DBClient {
     // Get deals
     const cids = uploads?.map((u) => u.content.cid)
     const deals = await this.getDealsForCids(cids)
-
     return {
       count,
       uploads: uploads?.map((u) => ({

packages/db/package.json

@@ -22,6 +22,7 @@
   "license": "(Apache-2.0 OR MIT)",
   "dependencies": {
     "@supabase/postgrest-js": "^0.37.0",
+    "multiformats": "^13.0.0",
     "p-retry": "^4.6.1"
   },
   "devDependencies": {

packages/db/postgres/functions.sql

@@ -10,6 +10,7 @@ DROP FUNCTION IF EXISTS user_used_storage;
 DROP FUNCTION IF EXISTS user_auth_keys_list;
 DROP FUNCTION IF EXISTS find_deals_by_content_cids;
 DROP FUNCTION IF EXISTS upsert_user;
+DROP TYPE IF EXISTS stored_bytes;
 
 -- transform a JSON array property into an array of SQL text elements
 CREATE OR REPLACE FUNCTION json_arr_to_text_arr(_json json)

packages/db/postgres/reset.sql

@@ -11,12 +11,18 @@ DROP TABLE IF EXISTS pin_sync_request;
 DROP TABLE IF EXISTS psa_pin_request;
 DROP TABLE IF EXISTS content;
 DROP TABLE IF EXISTS backup;
+DROP TABLE IF EXISTS auth_key_history;
 DROP TABLE IF EXISTS auth_key;
-DROP TABLE IF EXISTS public.user;
 DROP TABLE IF EXISTS user_tag;
 DROP TABLE IF EXISTS user_tag_proposal;
+DROP TABLE IF EXISTS email_history;
+DROP TABLE IF EXISTS user_customer;
+DROP TABLE IF EXISTS agreement;
+DROP TABLE IF EXISTS public.user;
 DROP TABLE IF EXISTS terms_of_service;
 
+DROP TYPE IF EXISTS stored_bytes;
+
 DROP SCHEMA IF EXISTS cargo CASCADE;
 DROP SERVER IF EXISTS dag_cargo_server CASCADE;
 DROP MATERIALIZED VIEW IF EXISTS public.aggregate_entry CASCADE;

packages/db/postgres/tables.sql

@@ -300,6 +300,7 @@ CREATE INDEX IF NOT EXISTS pin_sync_request_inserted_at_idx ON pin_sync_request
 
 -- Setting search_path to public scope for uuid function(s)
 SET search_path TO public;
+DROP TABLE IF EXISTS psa_pin_request;
 DROP extension IF EXISTS "uuid-ossp";
 CREATE extension "uuid-ossp" SCHEMA public;
 
@@ -356,6 +357,8 @@ CREATE TABLE IF NOT EXISTS email_history
   sent_at         TIMESTAMP WITH TIME ZONE DEFAULT timezone('utc'::text, now()) NOT NULL
 );
 
+
+DROP VIEW IF EXISTS admin_search;
 CREATE VIEW admin_search as
 select
   u.id::text as user_id,

packages/db/test/upload.spec.js

@@ -278,6 +278,41 @@ describe('upload', () => {
     assert.ok(userUploads.find(upload => upload.cid === sourceCid))
   })
 
+  it('lists user uploads with CAR links in parts', async () => {
+    const contentCid = 'bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi'
+    const sourceCid = 'QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR'
+    const exampleCarParkUrl = 'https://carpark-dev.web3.storage/bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea/bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea.car'
+    const exampleS3Url = 'https://dotstorage-dev-0.s3.us-east-1.amazonaws.com/raw/bafybeiao32xtnrlibcekpw3vyfi5txgrmvvrua4pccx3xik33ll3qhko2q/2/ciqplrl7tuebgpzbo5nqlqus5hj2kowxzz7ayr4z6ao2ftg7ibcr3ca.car'
+    const created = new Date().toISOString()
+    const name = `rand-${Math.random().toString().slice(2)}`
+    await client.createUpload({
+      user: user._id,
+      contentCid,
+      sourceCid,
+      authKey: authKeys[0]._id,
+      type,
+      dagSize,
+      name,
+      pins: [initialPinData],
+      backupUrls: [`https://backup.cid/${created}`, exampleCarParkUrl, exampleS3Url],
+      created
+    })
+
+    // Default sort {inserted_at, Desc}
+    const { uploads } = await client.listUploads(user._id, { page: 1 })
+    assert.ok(uploads.length > 0)
+    for (const upload of uploads) {
+      // backupUrls raw is private
+      assert.ok(!('backupUrls' in upload), 'upload does not have backupUrls property')
+      assert.ok(Array.isArray(upload.parts), 'upload.parts is an array')
+    }
+    const namedUpload = uploads.find(u => u.name === name)
+    assert.deepEqual(namedUpload.parts, [
+      // this corresponds to `exampleCarParkUrl`
+      'bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea'
+    ])
+  })
+
   it('can list user uploads with several options', async () => {
     const { uploads: previousUserUploads, count: previousUserUploadCount } = await client.listUploads(user._id, { page: 1 })
     assert(previousUserUploads, 'user has uploads')

packages/db/utils.js

@@ -1,3 +1,7 @@
+import * as Link from 'multiformats/link'
+import * as Digest from 'multiformats/hashes/digest'
+import { fromString } from 'uint8arrays'
+
 /**
  * Normalize upload item.
  *
@@ -6,17 +10,44 @@
  */
 export function normalizeUpload (upload) {
   const nUpload = { ...upload }
+  const backupUrls = nUpload.backupUrls ?? []
+  delete nUpload.backupUrls
   delete nUpload.content
   delete nUpload.sourceCid
 
+  /** @type {import('./db-client-types').UploadItemOutput['parts']} */
+  const parts = [...carCidV1Base32sFromBackupUrls(backupUrls)]
+
   return {
     ...nUpload,
     ...upload.content,
     cid: upload.sourceCid, // Overwrite cid to source cid
     pins: normalizePins(upload.content.pins, {
       isOkStatuses: true
-    })
+    }),
+    parts
+  }
+}
+
+/**
+ * given array of backup_urls from uploads table, return a corresponding set of CAR CIDv1 using base32 multihash
+ * for any CAR files in the backup_urls.
+ * @param {string[]} backupUrls
+ * @returns {Iterable<string>}
+ */
+function carCidV1Base32sFromBackupUrls (backupUrls) {
+  const carCidStrings = new Set()
+  for (const backupUrl of backupUrls) {
+    let carCid
+    try {
+      carCid = bucketKeyToPartCID(backupUrl)
+    } catch (error) {
+      console.warn('error extracting car CID from bucket URL', error)
+    }
+    if (!carCid) continue
+    carCidStrings.add(carCid.toString())
   }
+  return carCidStrings
 }
 
 /**
@@ -132,3 +163,30 @@ export function safeNumber (num) {
   }
   return num
 }
+
+const CAR_CODE = 0x0202
+
+/**
+ * Attempts to extract a CAR CID from a bucket key.
+ *
+ * @param {string} key
+ */
+const bucketKeyToPartCID = key => {
+  const filename = String(key.split('/').at(-1))
+  const [hash] = filename.split('.')
+  try {
+    // recent buckets encode CAR CID in filename
+    const cid = Link.parse(hash).toV1()
+    if (cid.code === CAR_CODE) return cid
+    throw new Error('not a CAR CID')
+  } catch (err) {
+    // older buckets base32 encode a CAR multihash <base32(car-multihash)>.car
+    try {
+      const digestBytes = fromString(hash, 'base32')
+      const digest = Digest.decode(digestBytes)
+      return Link.create(CAR_CODE, digest)
+    } catch (error) {
+      // console.warn('error trying to create CID from s3 key', error)
+    }
+  }
+}