Skip to content

Commit

Permalink
feat: blob implementation
Browse files Browse the repository at this point in the history
  • Loading branch information
vasco-santos committed Mar 25, 2024
1 parent 9aba8a1 commit 7660794
Show file tree
Hide file tree
Showing 19 changed files with 938 additions and 3 deletions.
3 changes: 2 additions & 1 deletion packages/capabilities/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,8 @@
"@ucanto/principal": "^9.0.0",
"@ucanto/transport": "^9.1.0",
"@ucanto/validator": "^9.0.1",
"@web3-storage/data-segment": "^3.2.0"
"@web3-storage/data-segment": "^3.2.0",
"uint8arrays": "^5.0.3"
},
"devDependencies": {
"@web3-storage/eslint-config-w3up": "workspace:^",
Expand Down
212 changes: 212 additions & 0 deletions packages/capabilities/src/blob.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
/**
* Blob Capabilities.
*
* Blob is a fixed size byte array addressed by the multihash.
* Usually blobs are used to represent set of IPLD blocks at different byte ranges.
*
* These can be imported directly with:
* ```js
* import * as Blob from '@web3-storage/capabilities/blob'
* ```
*
* @module
*/
import { capability, Link, Schema, ok, fail } from '@ucanto/validator'
import { equal, equalBlob, equalContent, equalWith, checkLink, SpaceDID, and } from './utils.js'

/**
* Agent capabilities for Blob protocol
*/

/**
* Capability can only be delegated (but not invoked) allowing audience to
* derived any `blob/` prefixed capability for the (memory) space identified
* by DID in the `with` field.
*/
export const blob = capability({
can: 'blob/*',
/**
* DID of the (memory) space where Blob is intended to
* be stored.
*/
with: SpaceDID,
derives: equalWith,
})

/**
* Blob description for being ingested by the service.
*/
export const blobStruct = Schema.struct({
/**
* A multihash digest of the blob payload bytes, uniquely identifying blob.
*/
content: Schema.bytes(),
/**
* Size of the Blob file to be stored. Service will provision write target
* for this exact size. Attempt to write a larger Blob file will fail.
*/
size: Schema.integer(),
})

/**
* `blob/add` capability allows agent to store a Blob into a (memory) space
* identified by did:key in the `with` field. Agent must precompute Blob locally
* and provide it's multihash and size using `nb.content` and `nb.size` fields, allowing
* a service to provision a write location for the agent to PUT or POST desired
* Blob into.
*/
export const add = capability({
can: 'blob/add',
/**
* DID of the (memory) space where Blob is intended to
* be stored.
*/
with: SpaceDID,
nb: Schema.struct({
/**
* Blob to allocate on the space.
*/
blob: blobStruct
}),
derives: equalBlob,
})

/**
* `blob/remove` capability can be used to remove the stored Blob from the (memory)
* space identified by `with` field.
*/
export const remove = capability({
can: 'blob/remove',
/**
* DID of the (memory) space where Blob is intended to
* be stored.
*/
with: SpaceDID,
nb: Schema.struct({
/**
* A multihash digest of the blob payload bytes, uniquely identifying blob.
*/
content: Schema.bytes(),
}),
derives: equalContent,
})

/**
* `blob/list` capability can be invoked to request a list of stored Blobs in the
* (memory) space identified by `with` field.
*/
export const list = capability({
can: 'blob/list',
/**
* DID of the (memory) space where Blob is intended to
* be stored.
*/
with: SpaceDID,
nb: Schema.struct({
/**
* A pointer that can be moved back and forth on the list.
* It can be used to paginate a list for instance.
*/
cursor: Schema.string().optional(),
/**
* Maximum number of items per page.
*/
size: Schema.integer().optional(),
/**
* If true, return page of results preceding cursor. Defaults to false.
*/
pre: Schema.boolean().optional(),
}),
derives: (claimed, delegated) => {
if (claimed.with !== delegated.with) {
return fail(
`Expected 'with: "${delegated.with}"' instead got '${claimed.with}'`
)
}
return ok({})
},
})

/**
* Service capabilities for Blob protocol
*/

// TODO: should we preffix these with some tmp service namespace that eases delegation of blob?
// OR
// export const blob = add.or(remove).or(list)

/**
* `blob/allocate` capability can be invoked to create a memory
* address where blob content can be written via HTTP PUT request.
*/
export const allocate = capability({
can: 'blob/allocate',
/**
* DID of the (memory) space where Blob is intended to
* be stored.
*/
with: SpaceDID,
nb: Schema.struct({
/**
* Blob to allocate on the space.
*/
blob: blobStruct,
/**
* The Link for an Add Blob task, that caused an allocation
*/
cause: Link,
/**
* DID of the user space where allocation takes place
*/
space: SpaceDID
}),
derives: (claim, from) => {
return (
and(equalWith(claim, from)) ||
and(equalBlob(claim, from)) ||
and(checkLink(claim.nb.cause, from.nb.cause, 'cause')) ||
and(equal(claim.nb.space, from.nb.space, 'space')) ||
ok({})
)
},
})

/**
* `blob/accept` capability invocation should either succeed when content is
* delivered on allocated address or fail if no content is allocation expires
* without content being delivered.
*/
export const accept = capability({
can: 'blob/accept',
/**
* DID of the (memory) space where Blob is intended to
* be stored.
*/
with: SpaceDID,
nb: Schema.struct({
/**
* Blob to accept.
*/
blob: blobStruct,
/**
* Expiration..
*/
exp: Schema.integer(),
}),
derives: (claim, from) => {
const result = equalBlob(claim, from)
if (result.error) {
return result
} else if (claim.nb.exp !== undefined && from.nb.exp !== undefined) {
return claim.nb.exp > from.nb.exp
? fail(`exp constraint violation: ${claim.nb.exp} > ${from.nb.exp}`)
: ok({})
} else {
return ok({})
}
},
})

// ⚠️ We export imports here so they are not omitted in generated typedes
// @see https://github.com/microsoft/TypeScript/issues/51548
export { Schema, Link }
7 changes: 7 additions & 0 deletions packages/capabilities/src/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import * as DealTracker from './filecoin/deal-tracker.js'
import * as UCAN from './ucan.js'
import * as Plan from './plan.js'
import * as Usage from './usage.js'
import * as Blob from './blob.js'

export {
Access,
Expand Down Expand Up @@ -86,4 +87,10 @@ export const abilitiesAsStrings = [
Plan.get.can,
Usage.usage.can,
Usage.report.can,
Blob.blob.can,
Blob.add.can,
Blob.remove.can,
Blob.list.can,
Blob.allocate.can,
Blob.accept.can,
]
75 changes: 74 additions & 1 deletion packages/capabilities/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import {
import { space, info } from './space.js'
import * as provider from './provider.js'
import { top } from './top.js'
import * as BlobCaps from './blob.js'
import * as StoreCaps from './store.js'
import * as UploadCaps from './upload.js'
import * as AccessCaps from './access.js'
Expand Down Expand Up @@ -439,6 +440,72 @@ export interface UploadNotFound extends Ucanto.Failure {

export type UploadGetFailure = UploadNotFound | Ucanto.Failure

// Blob
export type Blob = InferInvokedCapability<typeof BlobCaps.blob>
export type BlobAdd = InferInvokedCapability<typeof BlobCaps.add>
export type BlobRemove = InferInvokedCapability<typeof BlobCaps.remove>
export type BlobList = InferInvokedCapability<typeof BlobCaps.list>
export type BlobAllocate = InferInvokedCapability<typeof BlobCaps.allocate>
export type BlobAccept = InferInvokedCapability<typeof BlobCaps.accept>

export type BlobMultihash = Uint8Array

// Blob add
export interface BlobAddSuccess {
claim: {
'await/ok': Link
}
}
export type BlobAddFailure = Ucanto.Failure

// Blob remove
export interface BlobRemoveSuccess {
size: number
}

export interface BlobItemNotFound extends Ucanto.Failure {
name: 'BlobItemNotFound'
}

export type BlobRemoveFailure = BlobItemNotFound | Ucanto.Failure

// Blob list
export interface BlobListSuccess extends ListResponse<BlobListItem> {}
export interface BlobListItem {
blob: { content: Uint8Array, size: Number }
insertedAt: ISO8601Date
}

export type BlobListFailure = Ucanto.Failure

// Blob allocate
export interface BlobAllocateSuccess {
size: Number
address?: BlobAddress
}

export interface BlobAddress {
url: ToString<URL>
headers: Record<string, string>
}

export interface BlobItemNotFound extends Ucanto.Failure {
name: 'BlobItemNotFound'
}

export interface BlobNotAllocableToSpace extends Ucanto.Failure {
name: 'BlobNotAllocableToSpace'
}

export type BlobAllocateFailure = BlobItemNotFound | BlobNotAllocableToSpace | Ucanto.Failure

// Blob accept
export interface BlobAcceptSuccess {
claim: Link
}

export type BlobAcceptFailure = BlobItemNotFound | Ucanto.Failure

// Store
export type Store = InferInvokedCapability<typeof StoreCaps.store>
export type StoreAdd = InferInvokedCapability<typeof StoreCaps.add>
Expand Down Expand Up @@ -708,7 +775,13 @@ export type ServiceAbilityArray = [
AdminStoreInspect['can'],
PlanGet['can'],
Usage['can'],
UsageReport['can']
UsageReport['can'],
Blob['can'],
BlobAdd['can'],
BlobRemove['can'],
BlobList['can'],
BlobAllocate['can'],
BlobAccept['can'],
]

/**
Expand Down
Loading

0 comments on commit 7660794

Please sign in to comment.