From 8a79438bc82e0ef394344be90f2f075c892707fb Mon Sep 17 00:00:00 2001 From: adam-maj Date: Mon, 12 Sep 2022 18:18:41 -0700 Subject: [PATCH 01/19] Initial storage SDK update --- .../storage/src/{constants => common}/urls.ts | 0 .../{helpers/storage.ts => common/utils.ts} | 74 +++- .../core/downloaders/storage-downloader.ts | 7 + packages/storage/src/core/index.ts | 1 + packages/storage/src/core/ipfs-storage.ts | 356 ------------------ packages/storage/src/core/remote-storage.ts | 136 ------- packages/storage/src/core/storage.ts | 48 +++ .../src/core/uploaders/ipfs-uploader.ts | 16 + packages/storage/src/index.ts | 9 +- packages/storage/src/interfaces/IStorage.ts | 102 ----- .../storage/src/interfaces/IStorageUpload.ts | 44 --- packages/storage/src/types/data.ts | 55 +++ packages/storage/src/types/download.ts | 3 + packages/storage/src/types/events.ts | 44 --- packages/storage/src/types/index.ts | 35 +- packages/storage/src/types/upload.ts | 20 + .../storage/src/uploaders/pinata-uploader.ts | 187 --------- 17 files changed, 228 insertions(+), 909 deletions(-) rename packages/storage/src/{constants => common}/urls.ts (100%) rename packages/storage/src/{helpers/storage.ts => common/utils.ts} (66%) create mode 100644 packages/storage/src/core/downloaders/storage-downloader.ts create mode 100644 packages/storage/src/core/index.ts delete mode 100644 packages/storage/src/core/ipfs-storage.ts delete mode 100644 packages/storage/src/core/remote-storage.ts create mode 100644 packages/storage/src/core/storage.ts create mode 100644 packages/storage/src/core/uploaders/ipfs-uploader.ts delete mode 100644 packages/storage/src/interfaces/IStorage.ts delete mode 100644 packages/storage/src/interfaces/IStorageUpload.ts create mode 100644 packages/storage/src/types/data.ts create mode 100644 packages/storage/src/types/download.ts delete mode 100644 packages/storage/src/types/events.ts create mode 100644 packages/storage/src/types/upload.ts delete mode 100644 packages/storage/src/uploaders/pinata-uploader.ts diff --git a/packages/storage/src/constants/urls.ts b/packages/storage/src/common/urls.ts similarity index 100% rename from packages/storage/src/constants/urls.ts rename to packages/storage/src/common/urls.ts diff --git a/packages/storage/src/helpers/storage.ts b/packages/storage/src/common/utils.ts similarity index 66% rename from packages/storage/src/helpers/storage.ts rename to packages/storage/src/common/utils.ts index 1bc25438735..a1bf488b5a1 100644 --- a/packages/storage/src/helpers/storage.ts +++ b/packages/storage/src/common/utils.ts @@ -1,4 +1,8 @@ -import { Json } from "../types"; +import { IStorageUploader, Json, JsonObject } from "../types"; + +export function isBrowser() { + return typeof window !== "undefined"; +} export function isFileInstance(data: any): data is File { return global.File && data instanceof File; @@ -8,6 +12,74 @@ export function isBufferInstance(data: any): data is Buffer { return global.Buffer && data instanceof Buffer; } +/** + * Pre-processes metadata and uploads all file properties + * to storage in *bulk*, then performs a string replacement of + * all file properties -\> the resulting ipfs uri. This is + * called internally by `uploadMetadataBatch`. + * + * @internal + * + * @returns - The processed metadata with properties pointing at ipfs in place of `File | Buffer` + * @param metadatas + * @param options + */ +export async function batchUploadProperties( + metadatas: JsonObject[], + uploader: IStorageUploader, + gatewayUrl: string, + baseUri: string, +) { + // replace all active gateway url links with their raw ipfs hash + const sanitizedMetadatas = replaceGatewayUrlWithHash( + metadatas, + baseUri, + gatewayUrl, + ); + // extract any binary file to upload + const filesToUpload = sanitizedMetadatas.flatMap((m: JsonObject) => + buildFilePropertiesMap(m, []), + ); + // if no binary files to upload, return the metadata + if (filesToUpload.length === 0) { + return sanitizedMetadatas; + } + // otherwise upload those files + const uris = await uploader.uploadBatch(filesToUpload); + + // replace all files with their ipfs hash + return replaceFilePropertiesWithHashes(sanitizedMetadatas, uris, baseUri); +} + +/** + * This function recurisely traverses an object and hashes any + * `Buffer` or `File` objects into the returned map. + * + * @param object - the Json Object + * @param files - The running array of files or buffer to upload + * @returns - The final map of all hashes to files + */ +export function buildFilePropertiesMap( + object: JsonObject, + files: (File | Buffer)[] = [], +): (File | Buffer)[] { + if (Array.isArray(object)) { + object.forEach((element) => { + buildFilePropertiesMap(element, files); + }); + } else if (object) { + const values = Object.values(object); + for (const val of values) { + if (isFileInstance(val) || isBufferInstance(val)) { + files.push(val); + } else if (typeof val === "object") { + buildFilePropertiesMap(val as JsonObject, files); + } + } + } + return files; +} + /** * Given a map of file hashes to ipfs uris, this function will hash * all properties recursively and replace them with the ipfs uris diff --git a/packages/storage/src/core/downloaders/storage-downloader.ts b/packages/storage/src/core/downloaders/storage-downloader.ts new file mode 100644 index 00000000000..7bdb91115ac --- /dev/null +++ b/packages/storage/src/core/downloaders/storage-downloader.ts @@ -0,0 +1,7 @@ +import { IStorageDownloader } from "../../types"; + +export class StorageDownloader implements IStorageDownloader { + constructor() {} + + async download(url: string): Promise {} +} diff --git a/packages/storage/src/core/index.ts b/packages/storage/src/core/index.ts new file mode 100644 index 00000000000..433c831b99a --- /dev/null +++ b/packages/storage/src/core/index.ts @@ -0,0 +1 @@ +export { ThirdwebStorage } from "./storage"; diff --git a/packages/storage/src/core/ipfs-storage.ts b/packages/storage/src/core/ipfs-storage.ts deleted file mode 100644 index b799ceda44c..00000000000 --- a/packages/storage/src/core/ipfs-storage.ts +++ /dev/null @@ -1,356 +0,0 @@ -import { - DEFAULT_IPFS_GATEWAY, - PINATA_IPFS_URL, - PUBLIC_GATEWAYS, -} from "../constants/urls"; -import { - isBufferInstance, - isFileInstance, - replaceFilePropertiesWithHashes, - replaceGatewayUrlWithHash, - replaceHashWithGatewayUrl, - resolveGatewayUrl, -} from "../helpers/storage"; -import { IStorage } from "../interfaces/IStorage"; -import { IStorageUpload, UploadResult } from "../interfaces/IStorageUpload"; -import { FileOrBuffer, JsonObject, StorageOptions } from "../types"; -import { UploadProgressEvent } from "../types/events"; -import { PinataUploader } from "../uploaders/pinata-uploader"; -import fetch from "cross-fetch"; -import FormData from "form-data"; - -/** - * IPFS Storage implementation, accepts custom IPFS gateways - * @remarks By default, thirdweb automatically uploads files to IPFS when you perform operations such as minting, this class allows you to do it manually. - * @public - */ -export class IpfsStorage implements IStorage { - /** - * {@inheritdoc IStorage.gatewayUrl} - * @internal - */ - public gatewayUrl: string; - private failedUrls: string[] = []; - private uploader: IStorageUpload; - private options: StorageOptions | undefined; - - constructor( - gatewayUrl: string = DEFAULT_IPFS_GATEWAY, - uploader: IStorageUpload = new PinataUploader(), - options?: StorageOptions, - ) { - this.gatewayUrl = `${gatewayUrl.replace(/\/$/, "")}/`; - this.uploader = uploader; - this.options = options; - } - - private getNextPublicGateway() { - const urlsToTry = PUBLIC_GATEWAYS.filter( - (url) => !this.failedUrls.includes(url), - ).filter((url) => url !== this.gatewayUrl); - if (urlsToTry.length > 0) { - return urlsToTry[0]; - } else { - this.failedUrls = []; - return undefined; - } - } - - private getBaseUri() { - if (this.options?.appendGatewayUrl) { - return this.gatewayUrl; - } else { - return "ipfs://"; - } - } - - /** - * Upload a file to IPFS and return the hash - * @remarks This method is a wrapper around {@link IStorage.upload} - * @example - * ```javascript - * const file = './path/to/file.png'; // Can be a path or a File object such as a file from an input element. - * const hash = await sdk.storage.upload(file); - * ``` - * - * - */ - public async upload( - data: string | FileOrBuffer, - contractAddress?: string, - signerAddress?: string, - options?: { - onProgress: (event: UploadProgressEvent) => void; - }, - ): Promise { - const { cid, fileNames } = await this.uploader.uploadBatchWithCid( - [data], - 0, - contractAddress, - signerAddress, - options, - ); - - const baseUri = `${this.getBaseUri()}${cid}/`; - return `${baseUri}${fileNames[0]}`; - } - - /** - * {@inheritDoc IStorage.uploadBatch} - */ - public async uploadBatch( - files: (string | FileOrBuffer)[], - fileStartNumber = 0, - contractAddress?: string, - signerAddress?: string, - options?: { - onProgress: (event: UploadProgressEvent) => void; - }, - ) { - const { cid, fileNames } = await this.uploader.uploadBatchWithCid( - files, - fileStartNumber, - contractAddress, - signerAddress, - options, - ); - - const baseUri = `${this.getBaseUri()}${cid}/`; - const uris = fileNames.map((filename) => `${baseUri}${filename}`); - return { - baseUri, - uris, - }; - } - - /** - * {@inheritDoc IStorage.get} - */ - public async get(hash: string): Promise> { - const res = await this._get(hash); - const json = await res.json(); - return replaceHashWithGatewayUrl(json, "ipfs://", this.gatewayUrl); - } - - /** - * {@inheritDoc IStorage.getRaw} - */ - public async getRaw(hash: string): Promise { - const res = await this._get(hash); - return await res.text(); - } - - /** - * {@inheritDoc IStorage.uploadMetadata} - */ - public async uploadMetadata( - metadata: JsonObject, - contractAddress?: string, - signerAddress?: string, - options?: { - onProgress: (event: UploadProgressEvent) => void; - }, - ): Promise { - // since there's only single object, always use the first index - const { uris } = await this.uploadMetadataBatch( - [metadata], - 0, - contractAddress, - signerAddress, - options, - ); - return uris[0]; - } - - /** - * {@inheritDoc IStorage.uploadMetadataBatch} - */ - public async uploadMetadataBatch( - metadatas: JsonObject[], - fileStartNumber?: number, - contractAddress?: string, - signerAddress?: string, - options?: { - onProgress: (event: UploadProgressEvent) => void; - }, - ): Promise { - const metadataToUpload = ( - await this.batchUploadProperties(metadatas, options) - ).map((m: any) => JSON.stringify(m)); - - const { cid, fileNames } = await this.uploader.uploadBatchWithCid( - metadataToUpload, - fileStartNumber, - contractAddress, - signerAddress, - ); - - const baseUri = `${this.getBaseUri()}${cid}/`; - const uris = fileNames.map((filename) => `${baseUri}${filename}`); - - return { - baseUri, - uris, - }; - } - - /** ************************* - * PRIVATE FUNCTIONS - *************************/ - - private async _get(hash: string): Promise { - let uri = hash; - if (hash) { - uri = resolveGatewayUrl(hash, "ipfs://", this.gatewayUrl); - } - const result = await fetch(uri); - if (!result.ok && result.status === 500) { - throw new Error(`Error fetching ${uri} - Status code ${result.status}`); - } - if (!result.ok && result.status !== 404) { - const nextUrl = this.getNextPublicGateway(); - if (nextUrl) { - this.failedUrls.push(this.gatewayUrl); - this.gatewayUrl = nextUrl; - return this._get(hash); - } else { - throw new Error(`Error fetching ${uri} - Status code ${result.status}`); - } - } - return result; - } - - /** - * Pre-processes metadata and uploads all file properties - * to storage in *bulk*, then performs a string replacement of - * all file properties -\> the resulting ipfs uri. This is - * called internally by `uploadMetadataBatch`. - * - * @internal - * - * @returns - The processed metadata with properties pointing at ipfs in place of `File | Buffer` - * @param metadatas - * @param options - */ - private async batchUploadProperties( - metadatas: JsonObject[], - options?: { - onProgress: (event: UploadProgressEvent) => void; - }, - ) { - // replace all active gateway url links with their raw ipfs hash - const sanitizedMetadatas = replaceGatewayUrlWithHash( - metadatas, - this.getBaseUri(), - this.gatewayUrl, - ); - // extract any binary file to upload - const filesToUpload = sanitizedMetadatas.flatMap((m: JsonObject) => - this.buildFilePropertiesMap(m, []), - ); - // if no binary files to upload, return the metadata - if (filesToUpload.length === 0) { - return sanitizedMetadatas; - } - // otherwise upload those files - const { cid, fileNames } = await this.uploader.uploadBatchWithCid( - filesToUpload, - undefined, - undefined, - undefined, - options, - ); - - const cids = []; - // recurse ordered array - for (const filename of fileNames) { - cids.push(`${cid}/${filename}`); - } - - // replace all files with their ipfs hash - return replaceFilePropertiesWithHashes( - sanitizedMetadatas, - cids, - this.getBaseUri(), - ); - } - - /** - * This function recurisely traverses an object and hashes any - * `Buffer` or `File` objects into the returned map. - * - * @param object - the Json Object - * @param files - The running array of files or buffer to upload - * @returns - The final map of all hashes to files - */ - private buildFilePropertiesMap( - object: JsonObject, - files: (File | Buffer)[] = [], - ): (File | Buffer)[] { - if (Array.isArray(object)) { - object.forEach((element) => { - this.buildFilePropertiesMap(element, files); - }); - } else if (object) { - const values = Object.values(object); - for (const val of values) { - if (isFileInstance(val) || isBufferInstance(val)) { - files.push(val); - } else if (typeof val === "object") { - this.buildFilePropertiesMap(val as JsonObject, files); - } - } - } - return files; - } - - /** - * FOR TESTING ONLY - * @internal - * @param data - - * @param contractAddress - - * @param signerAddress - - */ - public async uploadSingle( - data: string | Record, - contractAddress?: string, - signerAddress?: string, - ): Promise { - // TODO move down to IStorageUpload - const token = await (this.uploader as PinataUploader).getUploadToken( - contractAddress || "", - ); - const metadata = { - name: `CONSOLE-TS-SDK-${contractAddress}`, - keyvalues: { - sdk: "typescript", - contractAddress, - signerAddress, - }, - }; - const formData = new FormData(); - const filepath = `files`; // Root directory - formData.append("file", data as any, filepath as any); - formData.append("pinataMetadata", JSON.stringify(metadata)); - formData.append( - "pinataOptions", - JSON.stringify({ - wrapWithDirectory: false, - }), - ); - const res = await fetch(PINATA_IPFS_URL, { - method: "POST", - headers: { - Authorization: `Bearer ${token}`, - ...formData.getHeaders(), - }, - body: formData.getBuffer(), - }); - if (!res.ok) { - throw new Error(`Failed to upload to IPFS [status code = ${res.status}]`); - } - - const body = await res.json(); - return body.IpfsHash; - } -} diff --git a/packages/storage/src/core/remote-storage.ts b/packages/storage/src/core/remote-storage.ts deleted file mode 100644 index d8be41d96fe..00000000000 --- a/packages/storage/src/core/remote-storage.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { isBufferInstance, isFileInstance } from "../helpers/storage"; -import { IStorage } from "../interfaces/IStorage"; -import { UploadResult } from "../interfaces/IStorageUpload"; -import { FileOrBuffer, JsonObject } from "../types"; -import { UploadProgressEvent } from "../types/events"; - -/** - * Fetch and upload files to IPFS or any other storage. - * @public - */ -export class RemoteStorage { - private storage: IStorage; - - constructor(storage: IStorage) { - this.storage = storage; - } - - /** - * Fetch data from any IPFS hash without worrying about gateways, data types, etc. - * Simply pass in an IPFS url and we'll handle fetching for you and try every public gateway - * to get the fastest response. - * - * @example - * ```javascript - * // Your IPFS hash here - * const hash = "ipfs://..." - * const data = await sdk.storage.fetch(hash); - * ``` - * @param hash - The IPFS hash of the file or data to fetch - * @returns The data stored at the specified IPFS hash - */ - public async fetch(hash: string): Promise> { - return this.storage.get(hash); - } - - /** - * Upload any data to an IPFS directory. We'll handle all the details for you, including - * pinning your files and making sure that you get the fastest upload speeds. - * - * @example - * ```javascript - * // File upload - * const files = [ - * fs.readFileSync("file1.png"), - * fs.readFileSync("file2.png"), - * ] - * const result = await sdk.storage.upload(files); - * // uri for each uploaded file will look like something like: ipfs:///0 - * - * // JSON metadata upload - * const jsonMetadata = { - * name: "Name", - * description: "Description", - * } - * const result = await sdk.storage.upload(jsonMetadata); - * - * // Upload progress (browser only) - * const result = await sdk.storage.upload(files, { - * onProgress: (event: UploadProgressEvent) => { - * console.log(`Downloaded ${event.progress} / ${event.total}`); - * }, - * }); - * ``` - * - * @param data - An array of file data or an array of JSON metadata to upload to IPFS - * @param options - Optional. Upload progress callback. - * @returns The IPFS hash of the directory that holds all the uploaded data - */ - public async upload( - data: FileOrBuffer[] | JsonObject[] | FileOrBuffer | JsonObject, - options?: { - onProgress: (event: UploadProgressEvent) => void; - }, - ): Promise { - if (!Array.isArray(data)) { - if ( - isFileInstance(data) || - isBufferInstance(data) || - (data.name && data.data && isBufferInstance(data.data)) - ) { - return this.uploadBatch([data as FileOrBuffer], options); - } else { - return this.uploadMetadataBatch([data as JsonObject], options); - } - } - - const allFiles = (data as any[]).filter( - (item: any) => - isFileInstance(item) || - isBufferInstance(item) || - (item.name && item.data && isBufferInstance(item.data)), - ); - const allObjects = (data as any[]).filter( - (item: any) => !isFileInstance(item) && !isBufferInstance(item), - ); - if (allFiles.length === data.length) { - return this.uploadBatch(data as FileOrBuffer[], options); - } else if (allObjects.length === data.length) { - return this.uploadMetadataBatch(data as JsonObject[], options); - } else { - throw new Error( - "Data to upload must be either all files or all JSON objects", - ); - } - } - - private async uploadBatch( - files: FileOrBuffer[], - options?: { - onProgress: (event: UploadProgressEvent) => void; - }, - ): Promise { - return await this.storage.uploadBatch( - files, - undefined, - undefined, - undefined, - options, - ); - } - - private async uploadMetadataBatch( - metadatas: JsonObject[], - options?: { - onProgress: (event: UploadProgressEvent) => void; - }, - ): Promise { - return await this.storage.uploadMetadataBatch( - metadatas, - undefined, - undefined, - undefined, - options, - ); - } -} diff --git a/packages/storage/src/core/storage.ts b/packages/storage/src/core/storage.ts new file mode 100644 index 00000000000..a093236e800 --- /dev/null +++ b/packages/storage/src/core/storage.ts @@ -0,0 +1,48 @@ +import { batchUploadProperties } from "../common/utils"; +import { + CleanedUploadData, + FileOrBufferSchema, + IStorageDownloader, + IStorageUploader, + UploadData, + UploadDataSchema, +} from "../types"; +import { StorageDownloader } from "./downloaders/storage-downloader"; +import { IpfsUploader } from "./uploaders/ipfs-uploader"; + +export class ThirdwebStorage { + private uploader: IStorageUploader; + private downloader: IStorageDownloader; + + constructor( + uploader: IStorageUploader = new IpfsUploader(), + downloader: IStorageDownloader = new StorageDownloader(), + ) { + this.uploader = uploader; + this.downloader = downloader; + } + + async download(url: string): Promise { + return this.downloader.download(url); + } + + async upload(data: UploadData): Promise { + const [uri] = await this.uploadBatch([data]); + return uri; + } + + async uploadBatch(data: UploadData[]): Promise { + const parsed = UploadDataSchema.parse(data); + const cleaned = parsed.map((item) => { + const { success } = FileOrBufferSchema.safeParse(item); + + if (success) { + return item; + } + + return JSON.stringify(item); + }) as CleanedUploadData[]; + + return this.uploader.uploadBatch(cleaned); + } +} diff --git a/packages/storage/src/core/uploaders/ipfs-uploader.ts b/packages/storage/src/core/uploaders/ipfs-uploader.ts new file mode 100644 index 00000000000..6b3c37b0bb6 --- /dev/null +++ b/packages/storage/src/core/uploaders/ipfs-uploader.ts @@ -0,0 +1,16 @@ +import { + FileOrBuffer, + IStorageUploader, + UploadProgressEvent, +} from "../../types"; + +export class IpfsUploader implements IStorageUploader { + constructor() {} + + async uploadBatch( + data: (string | FileOrBuffer)[], + onProgress?: (event: UploadProgressEvent) => void, + ): Promise { + return [""]; + } +} diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index fc3993acac2..d778e12df16 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -1,7 +1,2 @@ -export * from "./interfaces/IStorage"; -export * from "./interfaces/IStorageUpload"; -export * from "./core/ipfs-storage"; -export * from "./core/remote-storage"; -export * from "./uploaders/pinata-uploader"; -export * from "./types/index"; -export { isBufferInstance, isFileInstance } from "./helpers/storage"; +export * from "./core"; +export * from "./types"; diff --git a/packages/storage/src/interfaces/IStorage.ts b/packages/storage/src/interfaces/IStorage.ts deleted file mode 100644 index a7437a93cbe..00000000000 --- a/packages/storage/src/interfaces/IStorage.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { FileOrBuffer, JsonObject } from "../types"; -import { UploadProgressEvent } from "../types/events"; -import { UploadResult } from "./IStorageUpload"; - -/** - * Interface for any storage provider - * @public - */ -export interface IStorage { - /** - * Fetches data from storage. This method expects to fetch JSON formatted data - * - * @param hash - The Hash of the file to fetch - * @returns - The data, if found. - */ - get(hash: string): Promise>; - - /** - * Fetches data from storage. This method does not make any assumptions on the retrieved data format - * - * @param hash - The Hash of the file to fetch - * @returns - The data, if found. - */ - getRaw(hash: string): Promise; - - /** - * Uploads a file to the storage. - * - * @param data - The data to be uploaded. Can be a file/buffer (which will be loaded), or a string. - * @param contractAddress - Optional. The contract address the data belongs to. - * @param signerAddress - Optional. The address of the signer. - * @param options - Optional. Upload progress callback. - * @returns - The hash of the uploaded data. - */ - upload( - data: string | FileOrBuffer, - contractAddress?: string, - signerAddress?: string, - options?: { - onProgress: (event: UploadProgressEvent) => void; - }, - ): Promise; - - /** - * Uploads a folder to storage. - * - * @param files - An array of the data to be uploaded. Can be a files or buffers (which will be loaded), or strings. (can be mixed, too) - * @param fileStartNumber - Optional. The first file file name begins with. - * @param contractAddress - Optional. The contract address the data belongs to. - * @param signerAddress - Optional. The address of the signer. - * @param options - Optional. Upload progress callback. - * @returns - The CID of the uploaded folder. - */ - uploadBatch( - files: (string | FileOrBuffer)[], - fileStartNumber?: number, - contractAddress?: string, - signerAddress?: string, - options?: { - onProgress: (event: UploadProgressEvent) => void; - }, - ): Promise; - - /** - * - * Uploads JSON metadata to IPFS - * - * @param metadata - The metadata to be uploaded. - * @param contractAddress - Optional. The contract address the data belongs to. - * @param signerAddress - Optional. The address of the signer. - * @param options - Optional. Upload progress callback. - */ - - uploadMetadata( - metadata: JsonObject, - contractAddress?: string, - signerAddress?: string, - options?: { - onProgress: (event: UploadProgressEvent) => void; - }, - ): Promise; - - /** - * - * Uploads JSON metadata to IPFS - * - * @param metadatas - The metadata to be uploaded. - * @param fileStartNumber - Optional. The first file file name begins with. - * @param contractAddress - Optional. The contract address the data belongs to. - * @param signerAddress - Optional. The address of the signer. - * @param options - Optional. Upload progress callback. - */ - uploadMetadataBatch( - metadatas: JsonObject[], - fileStartNumber?: number, - contractAddress?: string, - signerAddress?: string, - options?: { - onProgress: (event: UploadProgressEvent) => void; - }, - ): Promise; -} diff --git a/packages/storage/src/interfaces/IStorageUpload.ts b/packages/storage/src/interfaces/IStorageUpload.ts deleted file mode 100644 index 895f5eb24ec..00000000000 --- a/packages/storage/src/interfaces/IStorageUpload.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { FileOrBuffer } from "../types"; -import { UploadProgressEvent } from "../types/events"; - -/** - * @internal - */ -export interface CidWithFileName { - // base cid of the directory - cid: string; - - // file name of the file without cid - fileNames: string[]; -} - -/** - * The result of an IPFS upload, including the URI of the upload - * director and the URIs of the uploaded files. - * @public - */ -export type UploadResult = { - /** - * Base URI of the directory that all files are uploaded to. - */ - baseUri: string; - /** - * Individual URI for each file or metadata upload. - */ - uris: string[]; -}; - -/** - * @internal - */ -export interface IStorageUpload { - uploadBatchWithCid( - files: (string | FileOrBuffer)[], - fileStartNumber?: number, - contractAddress?: string, - signerAddress?: string, - options?: { - onProgress: (event: UploadProgressEvent) => void; - }, - ): Promise; -} diff --git a/packages/storage/src/types/data.ts b/packages/storage/src/types/data.ts new file mode 100644 index 00000000000..a2c311c7547 --- /dev/null +++ b/packages/storage/src/types/data.ts @@ -0,0 +1,55 @@ +import { isBrowser } from "../common/utils"; +import { z } from "zod"; + +const JsonLiteralSchema = z.union([ + z.string(), + z.number(), + z.boolean(), + z.null(), +]); + +type JsonLiteral = z.infer; + +const FileOrBufferUnionSchema = isBrowser() + ? (z.instanceof(File) as z.ZodType>) + : (z.instanceof(Buffer) as z.ZodTypeAny); // TODO: fix, this is a hack to make browser happy for now + +export const FileOrBufferSchema = z.union([ + FileOrBufferUnionSchema, + z.object({ + data: z.union([z.instanceof(Buffer), z.string()]), + name: z.string(), + }), +]); + +export type FileOrBuffer = File | Buffer | BufferOrStringWithName; + +type BufferOrStringWithName = { + data: Buffer | string; + name: string; +}; + +const JsonSchema: z.ZodType = z.lazy(() => + z.union([ + JsonLiteralSchema, + JsonObjectSchema, + FileOrBufferSchema, + z.array(JsonSchema), + ]), +); + +export const JsonObjectSchema = z.record(z.string(), JsonSchema); + +export type Json = JsonLiteral | FileOrBuffer | JsonObject | Json[]; + +export type JsonObject = { [key: string]: Json }; + +export const UploadDataSchema = z.array( + z.union([z.array(JsonObjectSchema), z.array(FileOrBufferSchema)], { + invalid_type_error: "Must pass a file or buffer object or a JSON object.", + }), +); + +export type UploadData = JsonObject | FileOrBuffer; + +export type CleanedUploadData = string | FileOrBuffer; diff --git a/packages/storage/src/types/download.ts b/packages/storage/src/types/download.ts new file mode 100644 index 00000000000..9c8865f2cd0 --- /dev/null +++ b/packages/storage/src/types/download.ts @@ -0,0 +1,3 @@ +export interface IStorageDownloader { + download(url: string): Promise; +} diff --git a/packages/storage/src/types/events.ts b/packages/storage/src/types/events.ts deleted file mode 100644 index 01d69b7285e..00000000000 --- a/packages/storage/src/types/events.ts +++ /dev/null @@ -1,44 +0,0 @@ -export interface UploadProgressEvent { - /** - * The number of bytes uploaded. - */ - progress: number; - - /** - * The total number of bytes to be uploaded. - */ - total: number; -} - -/** - * Standardized return type for contract events that returns event arguments - */ -export type ContractEvent = { - eventName: string; - data: Record; - // from ethers.providers.Log - transaction: { - blockNumber: number; - blockHash: string; - transactionIndex: number; - - removed: boolean; - - address: string; - data: string; - - topics: Array; - - transactionHash: string; - logIndex: number; - }; -}; - -/** - * Filters for querying past events - */ -export interface EventQueryFilter { - fromBlock?: string | number; - toBlock?: string | number; - order?: "asc" | "desc"; -} diff --git a/packages/storage/src/types/index.ts b/packages/storage/src/types/index.ts index d4c506eafc5..41a7cb79d12 100644 --- a/packages/storage/src/types/index.ts +++ b/packages/storage/src/types/index.ts @@ -1,32 +1,3 @@ -import { z } from "zod"; - -export type BufferOrStringWithName = { - data: Buffer | string; - name?: string; -}; - -export type FileOrBuffer = File | Buffer | BufferOrStringWithName; - -type JsonLiteral = boolean | null | number | string; -type JsonLiteralOrFileOrBuffer = JsonLiteral | FileOrBuffer; -export type Json = JsonLiteralOrFileOrBuffer | JsonObject | Json[]; -export type JsonObject = { [key: string]: Json }; - -export const isBrowser = () => typeof window !== "undefined"; - -const fileOrBufferUnion = isBrowser() - ? ([z.instanceof(File), z.string()] as [ - z.ZodType>, - z.ZodString, - ]) - : ([z.instanceof(Buffer), z.string()] as [ - z.ZodTypeAny, // @fixme, this is a hack to make browser happy for now - z.ZodString, - ]); - -export const FileBufferOrStringSchema = z.union(fileOrBufferUnion); -export type FileBufferOrString = z.output; - -export type StorageOptions = { - appendGatewayUrl: boolean; -}; +export * from "./upload"; +export * from "./download"; +export * from "./data"; diff --git a/packages/storage/src/types/upload.ts b/packages/storage/src/types/upload.ts new file mode 100644 index 00000000000..fcc0f2a78a4 --- /dev/null +++ b/packages/storage/src/types/upload.ts @@ -0,0 +1,20 @@ +import { FileOrBuffer } from "./data"; + +export interface UploadProgressEvent { + /** + * The number of bytes uploaded. + */ + progress: number; + + /** + * The total number of bytes to be uploaded. + */ + total: number; +} + +export interface IStorageUploader { + uploadBatch( + data: (string | FileOrBuffer)[], + onProgress?: (event: UploadProgressEvent) => void, + ): Promise; +} diff --git a/packages/storage/src/uploaders/pinata-uploader.ts b/packages/storage/src/uploaders/pinata-uploader.ts deleted file mode 100644 index 27f776270c3..00000000000 --- a/packages/storage/src/uploaders/pinata-uploader.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { PINATA_IPFS_URL, TW_IPFS_SERVER_URL } from "../constants/urls"; -import { isBufferInstance, isFileInstance } from "../helpers/storage"; -import { CidWithFileName, IStorageUpload } from "../interfaces/IStorageUpload"; -import { FileOrBuffer } from "../types"; -import { UploadProgressEvent } from "../types/events"; -import fetch from "cross-fetch"; -import FormData from "form-data"; - -/** - * @internal - */ -export class PinataUploader implements IStorageUpload { - /** - * Fetches a one-time-use upload token that can used to upload - * a file to storage. - * - * @returns - The one time use token that can be passed to the Pinata API. - */ - public async getUploadToken(contractAddress: string): Promise { - const headers = { - "X-App-Name": `CONSOLE-TS-SDK-${contractAddress}`, - }; - const res = await fetch(`${TW_IPFS_SERVER_URL}/grant`, { - method: "GET", - headers, - }); - if (!res.ok) { - throw new Error(`Failed to get upload token`); - } - const body = await res.text(); - return body; - } - - public async uploadBatchWithCid( - files: (string | FileOrBuffer)[], - fileStartNumber = 0, - contractAddress?: string, - signerAddress?: string, - options?: { - onProgress: (event: UploadProgressEvent) => void; - }, - ): Promise { - const token = await this.getUploadToken(contractAddress || ""); - - const formData = new FormData(); - const { data, fileNames } = this.buildFormData( - formData, - files, - fileStartNumber, - contractAddress, - signerAddress, - ); - - if (typeof window === "undefined") { - if (options?.onProgress) { - console.warn("The onProgress option is only supported in the browser"); - } - const res = await fetch(PINATA_IPFS_URL, { - method: "POST", - headers: { - Authorization: `Bearer ${token}`, - ...data.getHeaders(), - }, - body: data.getBuffer(), - }); - const body = await res.json(); - if (!res.ok) { - throw new Error("Failed to upload files to IPFS"); - } - - const cid = body.IpfsHash; - if (!cid) { - throw new Error("Failed to upload files to IPFS"); - } - - return { - cid, - fileNames, - }; - } else { - return new Promise((resolve, reject) => { - const xhr = new XMLHttpRequest(); - xhr.open("POST", PINATA_IPFS_URL); - xhr.setRequestHeader("Authorization", `Bearer ${token}`); - - xhr.onloadend = () => { - if (xhr.status !== 200) { - throw new Error("Failed to upload files to IPFS"); - } - - const cid = JSON.parse(xhr.responseText).IpfsHash; - if (!cid) { - throw new Error("Failed to upload files to IPFS"); - } - - resolve({ - cid, - fileNames, - }); - }; - - xhr.onerror = (err) => { - reject(err); - }; - - if (xhr.upload) { - xhr.upload.onprogress = (event) => { - if (options?.onProgress) { - options?.onProgress({ - progress: event.loaded, - total: event.total, - }); - } - }; - } - - xhr.send(data); - }); - } - } - - private buildFormData( - data: any, - files: (string | FileOrBuffer)[], - fileStartNumber = 0, - contractAddress?: string, - signerAddress?: string, - ) { - const metadata = { - name: `CONSOLE-TS-SDK-${contractAddress}`, - keyvalues: { - sdk: "typescript", - contractAddress, - signerAddress, - }, - }; - - const fileNames: string[] = []; - files.forEach((file, i) => { - let fileName = ""; - let fileData = file; - // if it is a file, we passthrough the file extensions, - // if it is a buffer or string, the filename would be fileStartNumber + index - // if it is a buffer or string with names, the filename would be the name - if (isFileInstance(file)) { - let extensions = ""; - if (file.name) { - const extensionStartIndex = file.name.lastIndexOf("."); - if (extensionStartIndex > -1) { - extensions = file.name.substring(extensionStartIndex); - } - } - fileName = `${i + fileStartNumber}${extensions}`; - } else if (isBufferInstance(file) || typeof file === "string") { - fileName = `${i + fileStartNumber}`; - } else if (file && file.name && (file as any)?.data) { - fileData = (file as any)?.data; - fileName = `${file.name}`; - } else { - // default behavior - fileName = `${i + fileStartNumber}`; - } - - const filepath = `files/${fileName}`; - if (fileNames.indexOf(fileName) > -1) { - throw new Error( - `DUPLICATE_FILE_NAME_ERROR: File name ${fileName} was passed for more than one file.`, - ); - } - fileNames.push(fileName); - if (typeof window === "undefined") { - data.append("file", fileData as any, { filepath } as any); - } else { - // browser does blob things, filepath is parsed differently on browser vs node. - // pls pinata? - data.append("file", new Blob([fileData as any]), filepath); - } - }); - - data.append("pinataMetadata", JSON.stringify(metadata)); - - return { - data, - fileNames, - }; - } -} From 6a00d91f80e8e957e06b12ed6f2c996e893311fa Mon Sep 17 00:00:00 2001 From: adam-maj Date: Wed, 14 Sep 2022 15:07:57 -0700 Subject: [PATCH 02/19] Add storage downloader --- packages/storage/src/common/urls.ts | 17 +- packages/storage/src/common/utils.ts | 279 +++++------------- .../core/downloaders/storage-downloader.ts | 73 ++++- packages/storage/src/core/storage.ts | 43 ++- packages/storage/src/types/data.ts | 19 +- packages/storage/src/types/download.ts | 4 + 6 files changed, 195 insertions(+), 240 deletions(-) diff --git a/packages/storage/src/common/urls.ts b/packages/storage/src/common/urls.ts index eca5e785bcb..204b9c2a7e8 100644 --- a/packages/storage/src/common/urls.ts +++ b/packages/storage/src/common/urls.ts @@ -1,16 +1,15 @@ -/** - * @internal - */ -export const DEFAULT_IPFS_GATEWAY = "https://gateway.ipfscdn.io/ipfs/"; +import { GatewayUrls } from "../types"; /** * @internal */ -export const PUBLIC_GATEWAYS = [ - "https://gateway.ipfscdn.io/ipfs/", - "https://cloudflare-ipfs.com/ipfs/", - "https://ipfs.io/ipfs/", -]; +export const DEFAULT_GATEWAY_URLS: GatewayUrls = { + "ipfs://": [ + "https://gateway.ipfscdn.io/ipfs/", + "https://cloudflare-ipfs.com/ipfs/", + "https://ipfs.io/ipfs/", + ], +}; /** * @internal diff --git a/packages/storage/src/common/utils.ts b/packages/storage/src/common/utils.ts index a1bf488b5a1..871e77b3d1b 100644 --- a/packages/storage/src/common/utils.ts +++ b/packages/storage/src/common/utils.ts @@ -1,233 +1,100 @@ -import { IStorageUploader, Json, JsonObject } from "../types"; +import { + FileOrBuffer, + GatewayUrls, + IStorageUploader, + Json, + JsonObject, +} from "../types"; export function isBrowser() { return typeof window !== "undefined"; } -export function isFileInstance(data: any): data is File { +function isFileInstance(data: any): data is File { return global.File && data instanceof File; } -export function isBufferInstance(data: any): data is Buffer { +function isBufferInstance(data: any): data is Buffer { return global.Buffer && data instanceof Buffer; } -/** - * Pre-processes metadata and uploads all file properties - * to storage in *bulk*, then performs a string replacement of - * all file properties -\> the resulting ipfs uri. This is - * called internally by `uploadMetadataBatch`. - * - * @internal - * - * @returns - The processed metadata with properties pointing at ipfs in place of `File | Buffer` - * @param metadatas - * @param options - */ -export async function batchUploadProperties( - metadatas: JsonObject[], - uploader: IStorageUploader, - gatewayUrl: string, - baseUri: string, -) { - // replace all active gateway url links with their raw ipfs hash - const sanitizedMetadatas = replaceGatewayUrlWithHash( - metadatas, - baseUri, - gatewayUrl, - ); - // extract any binary file to upload - const filesToUpload = sanitizedMetadatas.flatMap((m: JsonObject) => - buildFilePropertiesMap(m, []), - ); - // if no binary files to upload, return the metadata - if (filesToUpload.length === 0) { - return sanitizedMetadatas; +export function replaceSchemeWithGatewayUrl( + uri: string, + gatewayUrls: GatewayUrls, + index = 0, +): string { + const scheme = Object.keys(gatewayUrls).find((s) => uri.startsWith(s)); + const schemeGatewayUrls = scheme ? gatewayUrls[scheme] : []; + + if (!scheme) { + return uri; + } + + if (index > schemeGatewayUrls.length) { + throw new Error( + "[GATEWAY_URL_ERROR] Failed to resolve gateway URL - ran out of gateway URLs to try.", + ); } - // otherwise upload those files - const uris = await uploader.uploadBatch(filesToUpload); - // replace all files with their ipfs hash - return replaceFilePropertiesWithHashes(sanitizedMetadatas, uris, baseUri); + return uri.replace(scheme, schemeGatewayUrls[index]); } -/** - * This function recurisely traverses an object and hashes any - * `Buffer` or `File` objects into the returned map. - * - * @param object - the Json Object - * @param files - The running array of files or buffer to upload - * @returns - The final map of all hashes to files - */ -export function buildFilePropertiesMap( - object: JsonObject, - files: (File | Buffer)[] = [], -): (File | Buffer)[] { - if (Array.isArray(object)) { - object.forEach((element) => { - buildFilePropertiesMap(element, files); - }); - } else if (object) { - const values = Object.values(object); - for (const val of values) { - if (isFileInstance(val) || isBufferInstance(val)) { - files.push(val); - } else if (typeof val === "object") { - buildFilePropertiesMap(val as JsonObject, files); +export function replaceObjectSchemesWithGatewayUrls( + data: Exclude, + gatewayUrls: GatewayUrls, +): Json { + switch (typeof data) { + case "string": + return replaceSchemeWithGatewayUrl(data, gatewayUrls); + case "object": + if (!data) { + return data; + } + + if (Array.isArray(data)) { + return data.map((entry) => + replaceObjectSchemesWithGatewayUrls( + entry as Exclude, + gatewayUrls, + ), + ); } - } - } - return files; -} -/** - * Given a map of file hashes to ipfs uris, this function will hash - * all properties recursively and replace them with the ipfs uris - * from the map passed in. If a hash is missing from the map, the function - * will throw an error. - * - * @internal - * - * @param object - The object to recursively process - * @param cids - The array of file hashes to ipfs uris in the recurse order - * @returns - The processed metadata with properties pointing at ipfs in place of `File | Buffer` - */ -export function replaceFilePropertiesWithHashes( - object: Record, - cids: string[], - scheme: string, -) { - const keys = Object.keys(object); - for (const key in keys) { - const val = object[keys[key]]; - const isFile = isFileInstance(val) || isBufferInstance(val); - if (typeof val === "object" && !isFile) { - replaceFilePropertiesWithHashes(val, cids, scheme); - continue; - } - - if (!isFile) { - continue; - } - - object[keys[key]] = `${scheme}${cids.splice(0, 1)[0]}`; + return Object.keys(data).map((key) => + replaceObjectSchemesWithGatewayUrls( + data[key] as Exclude, + gatewayUrls, + ), + ); } - return object; + + return data; } -/** - * Replaces all ipfs:// hashes (or any other scheme) with gateway url - * @internal - * @param object - * @param scheme - * @param gatewayUrl - */ -export function replaceHashWithGatewayUrl( - object: Record, - scheme: string, - gatewayUrl: string, -): Record { - if (object === null || !object) { - return {}; - } - const keys = Object.keys(object); - for (const key in keys) { - const val = object[keys[key]]; - object[keys[key]] = resolveGatewayUrl(val, scheme, gatewayUrl); - if (Array.isArray(val)) { - object[keys[key]] = val.map((el) => { - if (typeof el === "object") { - return replaceHashWithGatewayUrl(el, scheme, gatewayUrl); - } else { - return resolveGatewayUrl(el, scheme, gatewayUrl); - } - }); - } - if (typeof val === "object") { - replaceHashWithGatewayUrl(val, scheme, gatewayUrl); - } - } - return object; +export function replaceGatewayUrlWithScheme( + uri: string, + gatewayUrls: GatewayUrls, +): JsonObject[] { + return []; } -/** - * Replaces all gateway urls back to ipfs:// hashes - * @internal - * @param object - * @param scheme - * @param gatewayUrl - */ -export function replaceGatewayUrlWithHash( - object: Record, - scheme: string, - gatewayUrl: string, -): Record { - if (object === null || !object) { - return {}; - } - const keys = Object.keys(object); - for (const key in keys) { - const val = object[keys[key]]; - object[keys[key]] = toIPFSHash(val, scheme, gatewayUrl); - if (Array.isArray(val)) { - object[keys[key]] = val.map((el) => { - const isFile = isFileInstance(el) || isBufferInstance(el); - if (typeof el === "object" && !isFile) { - return replaceGatewayUrlWithHash(el, scheme, gatewayUrl); - } else { - return toIPFSHash(el, scheme, gatewayUrl); - } - }); - } - const isFile = isFileInstance(val) || isBufferInstance(val); - if (typeof val === "object" && !isFile) { - replaceGatewayUrlWithHash(val, scheme, gatewayUrl); - } - } - return object; +function extractFilesFromObjects(data: JsonObject[]): FileOrBuffer[] { + return []; } -/** - * Resolves the full URL of a file for a given gateway. - * - * For example, if the hash of a file is `ipfs://bafkreib3u2u6ir2fsl5nkuwixfsb3l4xehri3psjv5yga4inuzsjunk2sy`, then the URL will be: - * "https://cloudflare-ipfs.com/ipfs/bafkreibnwjhx5s3r2rggdoy3hw7lr7wmgy4bas35oky3ed6eijklk2oyvq" - * if the gateway is `cloudflare-ipfs.com`. - * @internal - * @param object - * @param scheme - * @param gatewayUrl - */ -export function resolveGatewayUrl( - object: T, - scheme: string, - gatewayUrl: string, -): T { - if (typeof object === "string") { - return object && object.toLowerCase().includes(scheme) - ? (object.replace(scheme, gatewayUrl) as T) - : object; - } else { - return object; - } +function replaceFilesWithHashes( + data: JsonObject[], + uris: string[], +): JsonObject[] { + return []; } -/** - * @internal - * @param object - * @param scheme - * @param gatewayUrl - */ -export function toIPFSHash( - object: T, - scheme: string, - gatewayUrl: string, -): T { - if (typeof object === "string") { - return object && object.toLowerCase().includes(gatewayUrl) - ? (object.replace(gatewayUrl, scheme) as T) - : object; - } else { - return object; - } +export async function uploadAndReplaceFilesWithHashes( + data: JsonObject[], +): Promise { + // Replace any gateway URLs with their hashes + // Recurse through data and extract files to upload + // Recurse through data and replace files with hashes + + return []; } diff --git a/packages/storage/src/core/downloaders/storage-downloader.ts b/packages/storage/src/core/downloaders/storage-downloader.ts index 7bdb91115ac..282037a35d1 100644 --- a/packages/storage/src/core/downloaders/storage-downloader.ts +++ b/packages/storage/src/core/downloaders/storage-downloader.ts @@ -1,7 +1,74 @@ -import { IStorageDownloader } from "../../types"; +import { DEFAULT_GATEWAY_URLS } from "../../common/urls"; +import { + replaceObjectSchemesWithGatewayUrls, + replaceSchemeWithGatewayUrl, +} from "../../common/utils"; +import { GatewayUrls, IStorageDownloader, Json } from "../../types"; export class StorageDownloader implements IStorageDownloader { - constructor() {} + public gatewayUrls: GatewayUrls; - async download(url: string): Promise {} + constructor(gatewayUrls?: GatewayUrls) { + this.gatewayUrls = this.prepareGatewayUrls(gatewayUrls); + } + + async download(uri: string, attempts = 0): Promise { + // Replace recognized scheme with the highest priority gateway URL that hasn't already been attempted + let resolvedUri; + try { + resolvedUri = replaceSchemeWithGatewayUrl( + uri, + this.gatewayUrls, + attempts, + ); + } catch (err: any) { + // If every gateway URL we know about for the designated scheme has been tried (via recursion) and failed, throw an error + if (err.includes("[GATEWAY_URL_ERROR]")) { + throw new Error( + "[FAILED_TO_DOWNLOAD_ERROR] Unable to download from URI - all gateway URLs failed to respond.", + ); + } + + throw err; + } + + const res = await fetch(resolvedUri); + + // If request to the current gateway fails, recursively try the next one we know about + if (!res.ok) { + console.warn( + `Request to ${resolvedUri} failed with status ${res.status} - ${res.statusText}`, + ); + return this.download(uri, attempts + 1); + } + + const text = await res.text(); + + // Handle both JSON and standard text data types + try { + // If we get a JSON object, recursively replace any schemes with gatewayUrls + const json = JSON.parse(text); + return replaceObjectSchemesWithGatewayUrls(json, this.gatewayUrls); + } catch { + return text; + } + } + + private prepareGatewayUrls(gatewayUrls?: GatewayUrls): GatewayUrls { + const allGatewayUrls = { + ...gatewayUrls, + ...DEFAULT_GATEWAY_URLS, + }; + + for (const key of Object.keys(DEFAULT_GATEWAY_URLS)) { + if (gatewayUrls && gatewayUrls[key]) { + allGatewayUrls[key] = [ + ...gatewayUrls[key], + ...DEFAULT_GATEWAY_URLS[key], + ]; + } + } + + return allGatewayUrls; + } } diff --git a/packages/storage/src/core/storage.ts b/packages/storage/src/core/storage.ts index a093236e800..b6edde724b3 100644 --- a/packages/storage/src/core/storage.ts +++ b/packages/storage/src/core/storage.ts @@ -1,10 +1,11 @@ import { batchUploadProperties } from "../common/utils"; import { CleanedUploadData, + FileOrBuffer, FileOrBufferSchema, IStorageDownloader, IStorageUploader, - UploadData, + JsonObject, UploadDataSchema, } from "../types"; import { StorageDownloader } from "./downloaders/storage-downloader"; @@ -26,23 +27,33 @@ export class ThirdwebStorage { return this.downloader.download(url); } - async upload(data: UploadData): Promise { - const [uri] = await this.uploadBatch([data]); + async upload(data: JsonObject | FileOrBuffer): Promise { + const [uri] = await this.uploadBatch([data] as + | JsonObject[] + | FileOrBuffer[]); return uri; } - async uploadBatch(data: UploadData[]): Promise { - const parsed = UploadDataSchema.parse(data); - const cleaned = parsed.map((item) => { - const { success } = FileOrBufferSchema.safeParse(item); - - if (success) { - return item; - } - - return JSON.stringify(item); - }) as CleanedUploadData[]; - - return this.uploader.uploadBatch(cleaned); + async uploadBatch(data: JsonObject[] | FileOrBuffer[]): Promise { + const parsed: JsonObject[] | FileOrBuffer[] = UploadDataSchema.parse(data); + const { success: isFileArray } = FileOrBufferSchema.safeParse(parsed[0]); + + // If data is an array of files, pass it through to upload directly + if (isFileArray) { + return this.uploader.uploadBatch(parsed as FileOrBuffer[]); + } + + // Otherwise it is an array of JSON objects, so we have to prepare it first + const metadata = ( + await batchUploadProperties( + parsed as JsonObject[], + this.uploader, + // TODO: Change baseUrl and gatewayUrl + "https://example.com", + "ipfs://", + ) + ).map((item) => JSON.stringify(item)); + + return this.uploader.uploadBatch(metadata); } } diff --git a/packages/storage/src/types/data.ts b/packages/storage/src/types/data.ts index a2c311c7547..9b38cee59d4 100644 --- a/packages/storage/src/types/data.ts +++ b/packages/storage/src/types/data.ts @@ -44,12 +44,19 @@ export type Json = JsonLiteral | FileOrBuffer | JsonObject | Json[]; export type JsonObject = { [key: string]: Json }; -export const UploadDataSchema = z.array( - z.union([z.array(JsonObjectSchema), z.array(FileOrBufferSchema)], { - invalid_type_error: "Must pass a file or buffer object or a JSON object.", - }), +export const UploadDataSchema = z.union( + [ + z.array(JsonObjectSchema).nonempty({ + message: "Cannot pass an empty array.", + }), + z.array(FileOrBufferSchema).nonempty({ + message: "Cannot pass an empty array.", + }), + ], + { + invalid_type_error: + "Must pass a an array of all files or buffer objects or an array of all JSON objects.", + }, ); -export type UploadData = JsonObject | FileOrBuffer; - export type CleanedUploadData = string | FileOrBuffer; diff --git a/packages/storage/src/types/download.ts b/packages/storage/src/types/download.ts index 9c8865f2cd0..a39b4bf063b 100644 --- a/packages/storage/src/types/download.ts +++ b/packages/storage/src/types/download.ts @@ -1,3 +1,7 @@ export interface IStorageDownloader { download(url: string): Promise; } + +export type GatewayUrls = { + [key: string]: string[]; +}; From 2aa7be4b47b8defb3bf9ceabb24971bc01b8c5f1 Mon Sep 17 00:00:00 2001 From: adam-maj Date: Wed, 14 Sep 2022 17:50:28 -0700 Subject: [PATCH 03/19] Add complete upload options --- packages/storage/src/common/urls.ts | 15 ++ packages/storage/src/common/utils.ts | 127 +++++++++--- .../core/downloaders/storage-downloader.ts | 43 +--- packages/storage/src/core/storage.ts | 81 ++++++-- .../src/core/uploaders/ipfs-uploader.ts | 190 +++++++++++++++++- packages/storage/src/types/data.ts | 2 +- packages/storage/src/types/download.ts | 3 +- packages/storage/src/types/upload.ts | 33 ++- 8 files changed, 405 insertions(+), 89 deletions(-) diff --git a/packages/storage/src/common/urls.ts b/packages/storage/src/common/urls.ts index 204b9c2a7e8..098c19df798 100644 --- a/packages/storage/src/common/urls.ts +++ b/packages/storage/src/common/urls.ts @@ -20,3 +20,18 @@ export const TW_IPFS_SERVER_URL = "https://upload.nftlabs.co"; * @internal */ export const PINATA_IPFS_URL = `https://api.pinata.cloud/pinning/pinFileToIPFS`; + +export function prepareGatewayUrls(gatewayUrls?: GatewayUrls): GatewayUrls { + const allGatewayUrls = { + ...gatewayUrls, + ...DEFAULT_GATEWAY_URLS, + }; + + for (const key of Object.keys(DEFAULT_GATEWAY_URLS)) { + if (gatewayUrls && gatewayUrls[key]) { + allGatewayUrls[key] = [...gatewayUrls[key], ...DEFAULT_GATEWAY_URLS[key]]; + } + } + + return allGatewayUrls; +} diff --git a/packages/storage/src/common/utils.ts b/packages/storage/src/common/utils.ts index 871e77b3d1b..553a2669c13 100644 --- a/packages/storage/src/common/utils.ts +++ b/packages/storage/src/common/utils.ts @@ -1,7 +1,8 @@ import { + BufferOrStringWithName, FileOrBuffer, + FileOrBufferSchema, GatewayUrls, - IStorageUploader, Json, JsonObject, } from "../types"; @@ -10,14 +11,35 @@ export function isBrowser() { return typeof window !== "undefined"; } -function isFileInstance(data: any): data is File { +export function isFileInstance(data: any): data is File { return global.File && data instanceof File; } -function isBufferInstance(data: any): data is Buffer { +export function isBufferInstance(data: any): data is Buffer { return global.Buffer && data instanceof Buffer; } +export function isBufferOrStringWithName( + data: any, +): data is BufferOrStringWithName { + return data && data.name && data.data; +} + +export function replaceGatewayUrlWithScheme( + uri: string, + gatewayUrls: GatewayUrls, +): string { + for (const scheme of Object.keys(gatewayUrls)) { + for (const url of gatewayUrls[scheme]) { + if (uri.startsWith(url)) { + return uri.replace(url, scheme); + } + } + } + + return uri; +} + export function replaceSchemeWithGatewayUrl( uri: string, gatewayUrls: GatewayUrls, @@ -39,6 +61,38 @@ export function replaceSchemeWithGatewayUrl( return uri.replace(scheme, schemeGatewayUrls[index]); } +export function replaceObjectGatewayUrlsWithSchemes( + data: Exclude, + gatewayUrls: GatewayUrls, +): Json { + switch (typeof data) { + case "string": + return replaceGatewayUrlWithScheme(data, gatewayUrls); + case "object": + if (!data) { + return data; + } + + if (Array.isArray(data)) { + return data.map((entry) => + replaceObjectGatewayUrlsWithSchemes( + entry as Exclude, + gatewayUrls, + ), + ); + } + + return Object.keys(data).map((key) => + replaceObjectGatewayUrlsWithSchemes( + data[key] as Exclude, + gatewayUrls, + ), + ); + } + + return data; +} + export function replaceObjectSchemesWithGatewayUrls( data: Exclude, gatewayUrls: GatewayUrls, @@ -71,30 +125,55 @@ export function replaceObjectSchemesWithGatewayUrls( return data; } -export function replaceGatewayUrlWithScheme( - uri: string, - gatewayUrls: GatewayUrls, -): JsonObject[] { - return []; -} +export function extractObjectFiles( + data: Json, + files: FileOrBuffer[] = [], +): FileOrBuffer[] { + // If item is a FileOrBuffer add it to our list of files + const { success } = FileOrBufferSchema.safeParse(data); + if (success) { + files.push(data as FileOrBuffer); + return files; + } -function extractFilesFromObjects(data: JsonObject[]): FileOrBuffer[] { - return []; -} + if (typeof data === "object") { + if (!data) { + return files; + } + + if (Array.isArray(data)) { + data.forEach((entry) => extractObjectFiles(entry, files)); + } + + Object.keys(data).map((key) => + extractObjectFiles((data as JsonObject)[key], files), + ); + } -function replaceFilesWithHashes( - data: JsonObject[], - uris: string[], -): JsonObject[] { - return []; + return files; } -export async function uploadAndReplaceFilesWithHashes( - data: JsonObject[], -): Promise { - // Replace any gateway URLs with their hashes - // Recurse through data and extract files to upload - // Recurse through data and replace files with hashes +export function replaceObjectFilesWithUris(data: Json, uris: string[]): Json { + // If item is a FileOrBuffer add it to our list of files + const { success: isFileOrBuffer } = FileOrBufferSchema.safeParse(data); + if (isFileOrBuffer) { + data = uris.splice(0, 1); + return data; + } + + if (typeof data === "object") { + if (!data) { + return data; + } - return []; + if (Array.isArray(data)) { + data.forEach((entry) => replaceObjectFilesWithUris(entry, uris)); + } + + Object.keys(data).map((key) => + replaceObjectFilesWithUris((data as JsonObject)[key], uris), + ); + } + + return data; } diff --git a/packages/storage/src/core/downloaders/storage-downloader.ts b/packages/storage/src/core/downloaders/storage-downloader.ts index 282037a35d1..cf7ced3e3d1 100644 --- a/packages/storage/src/core/downloaders/storage-downloader.ts +++ b/packages/storage/src/core/downloaders/storage-downloader.ts @@ -1,18 +1,16 @@ -import { DEFAULT_GATEWAY_URLS } from "../../common/urls"; -import { - replaceObjectSchemesWithGatewayUrls, - replaceSchemeWithGatewayUrl, -} from "../../common/utils"; -import { GatewayUrls, IStorageDownloader, Json } from "../../types"; +import { prepareGatewayUrls } from "../../common/urls"; +import { replaceSchemeWithGatewayUrl } from "../../common/utils"; +import { GatewayUrls, IStorageDownloader } from "../../types"; +import fetch from "cross-fetch"; export class StorageDownloader implements IStorageDownloader { public gatewayUrls: GatewayUrls; constructor(gatewayUrls?: GatewayUrls) { - this.gatewayUrls = this.prepareGatewayUrls(gatewayUrls); + this.gatewayUrls = prepareGatewayUrls(gatewayUrls); } - async download(uri: string, attempts = 0): Promise { + async download(uri: string, attempts = 0): Promise { // Replace recognized scheme with the highest priority gateway URL that hasn't already been attempted let resolvedUri; try { @@ -42,33 +40,6 @@ export class StorageDownloader implements IStorageDownloader { return this.download(uri, attempts + 1); } - const text = await res.text(); - - // Handle both JSON and standard text data types - try { - // If we get a JSON object, recursively replace any schemes with gatewayUrls - const json = JSON.parse(text); - return replaceObjectSchemesWithGatewayUrls(json, this.gatewayUrls); - } catch { - return text; - } - } - - private prepareGatewayUrls(gatewayUrls?: GatewayUrls): GatewayUrls { - const allGatewayUrls = { - ...gatewayUrls, - ...DEFAULT_GATEWAY_URLS, - }; - - for (const key of Object.keys(DEFAULT_GATEWAY_URLS)) { - if (gatewayUrls && gatewayUrls[key]) { - allGatewayUrls[key] = [ - ...gatewayUrls[key], - ...DEFAULT_GATEWAY_URLS[key], - ]; - } - } - - return allGatewayUrls; + return res; } } diff --git a/packages/storage/src/core/storage.ts b/packages/storage/src/core/storage.ts index b6edde724b3..df6b7fd2209 100644 --- a/packages/storage/src/core/storage.ts +++ b/packages/storage/src/core/storage.ts @@ -1,12 +1,18 @@ -import { batchUploadProperties } from "../common/utils"; import { - CleanedUploadData, + extractObjectFiles, + replaceObjectFilesWithUris, + replaceObjectGatewayUrlsWithSchemes, + replaceObjectSchemesWithGatewayUrls, +} from "../common/utils"; +import { FileOrBuffer, FileOrBufferSchema, IStorageDownloader, IStorageUploader, + Json, JsonObject, UploadDataSchema, + UploadOptions, } from "../types"; import { StorageDownloader } from "./downloaders/storage-downloader"; import { IpfsUploader } from "./uploaders/ipfs-uploader"; @@ -23,37 +29,82 @@ export class ThirdwebStorage { this.downloader = downloader; } - async download(url: string): Promise { - return this.downloader.download(url); + async download(url: string): Promise { + const res = await this.downloader.download(url); + + const text = await res.text(); + + // Handle both JSON and standard text data types + try { + // If we get a JSON object, recursively replace any schemes with gatewayUrls + const json = JSON.parse(text); + return replaceObjectSchemesWithGatewayUrls( + json, + this.downloader.gatewayUrls, + ); + } catch { + return text; + } } - async upload(data: JsonObject | FileOrBuffer): Promise { + async upload( + data: JsonObject | FileOrBuffer, + options?: UploadOptions, + ): Promise { const [uri] = await this.uploadBatch([data] as | JsonObject[] | FileOrBuffer[]); return uri; } - async uploadBatch(data: JsonObject[] | FileOrBuffer[]): Promise { + async uploadBatch( + data: JsonObject[] | FileOrBuffer[], + options?: UploadOptions, + ): Promise { const parsed: JsonObject[] | FileOrBuffer[] = UploadDataSchema.parse(data); const { success: isFileArray } = FileOrBufferSchema.safeParse(parsed[0]); // If data is an array of files, pass it through to upload directly if (isFileArray) { - return this.uploader.uploadBatch(parsed as FileOrBuffer[]); + return this.uploader.uploadBatch(parsed as FileOrBuffer[], options); } // Otherwise it is an array of JSON objects, so we have to prepare it first const metadata = ( - await batchUploadProperties( - parsed as JsonObject[], - this.uploader, - // TODO: Change baseUrl and gatewayUrl - "https://example.com", - "ipfs://", - ) + await this.uploadAndReplaceFilesWithHashes(parsed as JsonObject[]) ).map((item) => JSON.stringify(item)); - return this.uploader.uploadBatch(metadata); + return this.uploader.uploadBatch(metadata, options); + } + + private async uploadAndReplaceFilesWithHashes( + data: JsonObject[], + ): Promise { + let cleaned = data; + if (this.downloader.gatewayUrls) { + // Replace any gateway URLs with their hashes + cleaned = replaceObjectGatewayUrlsWithSchemes( + data, + this.downloader.gatewayUrls, + ) as JsonObject[]; + } + + if (this.uploader.uploadWithGatewayUrl) { + // If flag is set, replace all schemes with their preferred gateway URL + // Ex: used for Solana, where services don't resolve schemes for you, so URLs must be useable by default + cleaned = replaceObjectSchemesWithGatewayUrls( + data, + this.uploader.gatewayUrls || this.downloader.gatewayUrls, + ) as JsonObject[]; + } + + // Recurse through data and extract files to upload + const files = extractObjectFiles(data); + + // Upload all files that came from the object + const uris = await this.uploader.uploadBatch(files); + + // Recurse through data and replace files with hashes + return replaceObjectFilesWithUris(data, uris) as JsonObject[]; } } diff --git a/packages/storage/src/core/uploaders/ipfs-uploader.ts b/packages/storage/src/core/uploaders/ipfs-uploader.ts index 6b3c37b0bb6..8a32b229e7b 100644 --- a/packages/storage/src/core/uploaders/ipfs-uploader.ts +++ b/packages/storage/src/core/uploaders/ipfs-uploader.ts @@ -1,16 +1,198 @@ +import { + PINATA_IPFS_URL, + prepareGatewayUrls, + TW_IPFS_SERVER_URL, +} from "../../common/urls"; +import { + isBrowser, + isBufferOrStringWithName, + isFileInstance, +} from "../../common/utils"; import { FileOrBuffer, + GatewayUrls, + IpfsUploadBatchOptions, + IpfsUploaderOptions, IStorageUploader, - UploadProgressEvent, } from "../../types"; +import fetch from "cross-fetch"; +import FormData from "form-data"; export class IpfsUploader implements IStorageUploader { - constructor() {} + public gatewayUrls: GatewayUrls; + public uploadWithGatewayUrl: boolean; + + constructor(options?: IpfsUploaderOptions) { + this.gatewayUrls = prepareGatewayUrls(options?.gatewayUrls); + this.uploadWithGatewayUrl = options?.uploadWithGatewayUrl || false; + } async uploadBatch( data: (string | FileOrBuffer)[], - onProgress?: (event: UploadProgressEvent) => void, + options?: IpfsUploadBatchOptions, + ): Promise { + const form = new FormData(); + const { fileNames } = this.buildFormData(form, data); + + if (isBrowser()) { + return this.uploadBatchBrowser(form, fileNames, options); + } else { + return this.uploadBatchNode(form, fileNames, options); + } + } + + /** + * Fetches a one-time-use upload token that can used to upload + * a file to storage. + * + * @returns - The one time use token that can be passed to the Pinata API. + */ + private async getUploadToken(): Promise { + const res = await fetch(`${TW_IPFS_SERVER_URL}/grant`, { + method: "GET", + }); + if (!res.ok) { + throw new Error(`Failed to get upload token`); + } + const body = await res.text(); + return body; + } + + private buildFormData( + form: FormData, + files: (string | FileOrBuffer)[], + options?: IpfsUploadBatchOptions, + ) { + const fileNames: string[] = []; + files.forEach((file, i) => { + let fileName = ""; + let fileData = file; + + if (isFileInstance(file)) { + if (options?.rewriteFileNames) { + let extensions = ""; + if (file.name) { + const extensionStartIndex = file.name.lastIndexOf("."); + if (extensionStartIndex > -1) { + extensions = file.name.substring(extensionStartIndex); + } + } + fileName = `${ + i + options.rewriteFileNames.fileStartNumber + }${extensions}`; + } else { + fileName = `${file.name}`; + } + } else if (isBufferOrStringWithName(file)) { + fileData = file.data; + if (options?.rewriteFileNames) { + fileName = `${i + options.rewriteFileNames.fileStartNumber}`; + } else { + fileName = `${file.name}`; + } + } else { + if (options?.rewriteFileNames) { + fileName = `${i + options.rewriteFileNames.fileStartNumber}`; + } else { + fileName = `${i}`; + } + } + + const filepath = `files/${fileName}`; + if (fileNames.indexOf(fileName) > -1) { + throw new Error( + `[DUPLICATE_FILE_NAME_ERROR] File name ${fileName} was passed for more than one file.`, + ); + } + + fileNames.push(fileName); + if (!isBrowser()) { + form.append("file", fileData as any, { filepath } as any); + } else { + // browser does blob things, filepath is parsed differently on browser vs node. + // pls pinata? + form.append("file", new Blob([fileData as any]), filepath); + } + }); + + return { + form, + fileNames, + }; + } + + private async uploadBatchBrowser( + form: FormData, + fileNames: string[], + options?: IpfsUploadBatchOptions, ): Promise { - return [""]; + const token = await this.getUploadToken(); + + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.open("POST", PINATA_IPFS_URL); + xhr.setRequestHeader("Authorization", `Bearer ${token}`); + + xhr.onloadend = () => { + if (xhr.status !== 200) { + throw new Error("Failed to upload files to IPFS"); + } + + const cid = JSON.parse(xhr.responseText).IpfsHash; + if (!cid) { + throw new Error("Failed to upload files to IPFS"); + } + + resolve(fileNames.map((name) => `ipfs://${cid}/${name}`)); + }; + + xhr.onerror = (err) => { + reject(err); + }; + + if (xhr.upload) { + xhr.upload.onprogress = (event) => { + if (options?.onProgress) { + options?.onProgress({ + progress: event.loaded, + total: event.total, + }); + } + }; + } + + xhr.send(form as any); + }); + } + + private async uploadBatchNode( + form: FormData, + fileNames: string[], + options?: IpfsUploadBatchOptions, + ) { + const token = await this.getUploadToken(); + + if (options?.onProgress) { + console.warn("The onProgress option is only supported in the browser"); + } + const res = await fetch(PINATA_IPFS_URL, { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + ...form.getHeaders(), + }, + body: form.getBuffer(), + }); + const body = await res.json(); + if (!res.ok) { + throw new Error("Failed to upload files to IPFS"); + } + + const cid = body.IpfsHash; + if (!cid) { + throw new Error("Failed to upload files to IPFS"); + } + + return fileNames.map((name) => `ipfs://${cid}/${name}`); } } diff --git a/packages/storage/src/types/data.ts b/packages/storage/src/types/data.ts index 9b38cee59d4..084bb1b94ef 100644 --- a/packages/storage/src/types/data.ts +++ b/packages/storage/src/types/data.ts @@ -24,7 +24,7 @@ export const FileOrBufferSchema = z.union([ export type FileOrBuffer = File | Buffer | BufferOrStringWithName; -type BufferOrStringWithName = { +export type BufferOrStringWithName = { data: Buffer | string; name: string; }; diff --git a/packages/storage/src/types/download.ts b/packages/storage/src/types/download.ts index a39b4bf063b..78d3bb25fc1 100644 --- a/packages/storage/src/types/download.ts +++ b/packages/storage/src/types/download.ts @@ -1,5 +1,6 @@ export interface IStorageDownloader { - download(url: string): Promise; + gatewayUrls: GatewayUrls; + download(url: string): Promise; } export type GatewayUrls = { diff --git a/packages/storage/src/types/upload.ts b/packages/storage/src/types/upload.ts index fcc0f2a78a4..72a60f05809 100644 --- a/packages/storage/src/types/upload.ts +++ b/packages/storage/src/types/upload.ts @@ -1,6 +1,18 @@ import { FileOrBuffer } from "./data"; +import { GatewayUrls } from "./download"; -export interface UploadProgressEvent { +export type UploadOptions = { [key: string]: any }; + +export interface IStorageUploader { + gatewayUrls?: GatewayUrls; + uploadWithGatewayUrl?: boolean; + uploadBatch( + data: (string | FileOrBuffer)[], + options?: UploadOptions, + ): Promise; +} + +export type UploadProgressEvent = { /** * The number of bytes uploaded. */ @@ -10,11 +22,16 @@ export interface UploadProgressEvent { * The total number of bytes to be uploaded. */ total: number; -} +}; -export interface IStorageUploader { - uploadBatch( - data: (string | FileOrBuffer)[], - onProgress?: (event: UploadProgressEvent) => void, - ): Promise; -} +export type IpfsUploaderOptions = { + gatewayUrls?: GatewayUrls; + uploadWithGatewayUrl?: boolean; +}; + +export type IpfsUploadBatchOptions = { + rewriteFileNames?: { + fileStartNumber: number; + }; + onProgress?: (event: UploadProgressEvent) => void; +}; From 0830ccb92fa042c84b0635cf060db19952118178 Mon Sep 17 00:00:00 2001 From: adam-maj Date: Wed, 14 Sep 2022 18:08:21 -0700 Subject: [PATCH 04/19] Update IPFS tests --- packages/storage/src/core/storage.ts | 2 +- packages/storage/test/ipfs.test.ts | 103 +++++++++++++-------------- 2 files changed, 49 insertions(+), 56 deletions(-) diff --git a/packages/storage/src/core/storage.ts b/packages/storage/src/core/storage.ts index df6b7fd2209..bcea5be8d4d 100644 --- a/packages/storage/src/core/storage.ts +++ b/packages/storage/src/core/storage.ts @@ -29,7 +29,7 @@ export class ThirdwebStorage { this.downloader = downloader; } - async download(url: string): Promise { + async download(url: string): Promise { const res = await this.downloader.download(url); const text = await res.text(); diff --git a/packages/storage/test/ipfs.test.ts b/packages/storage/test/ipfs.test.ts index b62a557c8f9..88a7aba097a 100644 --- a/packages/storage/test/ipfs.test.ts +++ b/packages/storage/test/ipfs.test.ts @@ -1,18 +1,20 @@ -import { PUBLIC_GATEWAYS, DEFAULT_IPFS_GATEWAY } from "../src/constants/urls"; -import { IpfsStorage } from "../src/core/ipfs-storage"; -import { RemoteStorage } from "../src/core/remote-storage"; -import { BufferOrStringWithName, FileOrBuffer } from "../src/types"; +import { ThirdwebStorage } from "../src"; +import { DEFAULT_GATEWAY_URLS } from "../src/common/urls"; +import { + BufferOrStringWithName, + FileOrBuffer, + IpfsUploadBatchOptions, +} from "../src/types"; import { assert, expect } from "chai"; import fetch from "cross-fetch"; import { readFileSync } from "fs"; -const ipfsGatewayUrl = DEFAULT_IPFS_GATEWAY; +const ipfsGatewayUrl = DEFAULT_GATEWAY_URLS["ipfs://"][0]; global.fetch = require("cross-fetch"); describe("IPFS Uploads", async () => { - const storage: IpfsStorage = new IpfsStorage(ipfsGatewayUrl); - const remoteStorage: RemoteStorage = new RemoteStorage(storage); + const storage = new ThirdwebStorage(); async function getFile(upload: string): Promise { const response = await fetch( @@ -30,21 +32,19 @@ describe("IPFS Uploads", async () => { describe("SDK storage interface", async () => { it("Should single upload and fetch", async () => { - const { - uris: [uri], - } = await remoteStorage.upload({ + const uri = await storage.upload({ name: "Test Name", description: "Test Description", }); - const data = await remoteStorage.fetch(uri); + const data = await storage.download(uri); expect(data.name).to.equal("Test Name"); expect(data.description).to.equal("Test Description"); }); it("Should batch upload and fetch", async () => { - const { uris } = await remoteStorage.upload([ + const uris = await storage.uploadBatch([ { name: "Test Name 1", description: "Test Description 1", @@ -55,8 +55,8 @@ describe("IPFS Uploads", async () => { }, ]); - const data1 = await remoteStorage.fetch(uris[0]); - const data2 = await remoteStorage.fetch(uris[1]); + const data1 = await storage.download(uris[0]); + const data2 = await storage.download(uris[1]); expect(data1.name).to.equal("Test Name 1"); expect(data1.description).to.equal("Test Description 1"); @@ -72,21 +72,20 @@ describe("IPFS Uploads", async () => { const metadata = JSON.parse( readFileSync("test/files/metadata.json", "utf8"), ); - const upload = await storage.uploadMetadataBatch([metadata]); - assert.isTrue(upload.uris.length > 0); + const uris = await storage.uploadBatch([metadata]); + assert.isTrue(uris.length > 0); } catch (err) { assert.fail(err as string); } }); it("should upload metadata and replace gateway urls on upload/download", async () => { - const upload = await storage.uploadMetadata({ + const upload = await storage.upload({ svg: `${ipfsGatewayUrl}QmZsU8nTTexTxPzCKZKqo3Ntf5cUiWMRahoLmtpimeaCiT/backgrounds/SVG/Asset%20501.svg`, }); - const otherStorage = new IpfsStorage(PUBLIC_GATEWAYS[1]); - const downloaded = await otherStorage.get(upload); + const downloaded = await storage.download(upload); expect(downloaded.svg).to.eq( - `${PUBLIC_GATEWAYS[1]}QmZsU8nTTexTxPzCKZKqo3Ntf5cUiWMRahoLmtpimeaCiT/backgrounds/SVG/Asset%20501.svg`, + `${DEFAULT_GATEWAY_URLS[1]}QmZsU8nTTexTxPzCKZKqo3Ntf5cUiWMRahoLmtpimeaCiT/backgrounds/SVG/Asset%20501.svg`, ); }); @@ -98,9 +97,6 @@ describe("IPFS Uploads", async () => { readFileSync("test/files/1.jpg"), readFileSync("test/files/test.mp4"), ], - undefined, - undefined, - undefined, { onProgress: () => { updates += 1; @@ -113,7 +109,7 @@ describe("IPFS Uploads", async () => { it("should upload a file through any property, even when it is in an object nested inside another object", async () => { try { - const upload = await storage.uploadMetadata({ + const upload = await storage.upload({ name: "test", image: readFileSync("test/files/bone.jpg"), test: { @@ -134,7 +130,7 @@ describe("IPFS Uploads", async () => { }); it("should not upload the string to IPFS", async () => { - const upload = await storage.uploadMetadata({ + const upload = await storage.upload({ image: "ipfs://QmZsU8nTTexTxPzCKZKqo3Ntf5cUiWMRahoLmtpimeaCiT/face_parts/Asset%20331.svg", }); @@ -145,7 +141,7 @@ describe("IPFS Uploads", async () => { }); it("should upload an MP4 file when passed in the animation_url property", async () => { - const upload = await storage.uploadMetadata({ + const upload = await storage.upload({ animation_url: readFileSync("test/files/test.mp4"), }); assert.equal( @@ -155,14 +151,14 @@ describe("IPFS Uploads", async () => { }); it("should upload an media file and resolve to gateway URL when fetching it", async () => { - const upload = await storage.uploadMetadata({ + const upload = await storage.upload({ animation_url: readFileSync("test/files/test.mp4"), }); assert.equal( upload, "ipfs://QmbaNzUcv7KPgdwq9u2qegcptktpUK6CdRZF72eSjSa6iJ/0", ); - const meta = await storage.get(upload); + const meta = await storage.download(upload); assert.equal( meta.animation_url, `${ipfsGatewayUrl}QmUphf8LnNGdFwBevnxNkq8dxcZ4qxzzPjoNMDkSQfECKM/0`, @@ -186,9 +182,14 @@ describe("IPFS Uploads", async () => { const serialized = sampleObjects.map((o) => Buffer.from(JSON.stringify(o)), ); - const { baseUri: cid } = await storage.uploadBatch(serialized); + const uris = await storage.uploadBatch(serialized, { + rewriteFileNames: { + fileStartNumber: 1, + }, + } as IpfsUploadBatchOptions); for (const object of sampleObjects) { - const parsed = await storage.get(`${cid}${object.id}`); + assert(uris[object.id].endsWith(object.id.toString())); + const parsed = await storage.download(uris[object.id]); assert.equal(parsed.description, object.description); assert.equal(parsed.id, object.id); } @@ -206,12 +207,10 @@ describe("IPFS Uploads", async () => { name: "test.jpeg", }, ]; - const { baseUri: cid } = await storage.uploadBatch(sampleObjects); + const uris = await storage.uploadBatch(sampleObjects); assert( - (await getFile(`${cid}test.jpeg`)).headers - ?.get("content-type") - ?.toString() === "image/jpeg", - `${cid}`, + (await getFile(uris[2])).headers?.get("content-type")?.toString() === + "image/jpeg", ); }); @@ -220,11 +219,14 @@ describe("IPFS Uploads", async () => { readFileSync("test/files/test.mp4"), readFileSync("test/files/bone.jpg"), ]; - const { baseUri: cid } = await storage.uploadBatch(sampleObjects, 1); + const uris = await storage.uploadBatch(sampleObjects, { + rewriteFileNames: { + fileStartNumber: 1, + }, + } as IpfsUploadBatchOptions); assert( - (await getFile(`${cid}2`)).headers?.get("content-type")?.toString() === + (await getFile(uris[1])).headers?.get("content-type")?.toString() === "image/jpeg", - `${cid}`, ); }); it("should upload files according to start file number as 0", async () => { @@ -232,11 +234,10 @@ describe("IPFS Uploads", async () => { readFileSync("test/files/bone.jpg"), readFileSync("test/files/test.mp4"), ]; - const { baseUri: cid } = await storage.uploadBatch(sampleObjects); + const uris = await storage.uploadBatch(sampleObjects); assert( - (await getFile(`${cid}0`)).headers?.get("content-type")?.toString() === + (await getFile(uris[0])).headers?.get("content-type")?.toString() === "image/jpeg", - `${cid}`, ); }); it("should upload properties recursively in batch", async () => { @@ -263,10 +264,7 @@ describe("IPFS Uploads", async () => { }, }, ]; - const { baseUri, uris } = await storage.uploadMetadataBatch( - sampleObjects, - ); - assert(baseUri.startsWith("ipfs://") && baseUri.endsWith("/")); + const uris = await storage.uploadBatch(sampleObjects); assert(uris.length === sampleObjects.length); const [metadata1, metadata2, metadata3] = await Promise.all( ( @@ -296,13 +294,10 @@ describe("IPFS Uploads", async () => { }; sampleObjects.push(nft); } - const { baseUri, uris } = await storage.uploadMetadataBatch( - sampleObjects, - ); - assert(baseUri.startsWith("ipfs://") && baseUri.endsWith("/")); + const uris = await storage.uploadBatch(sampleObjects); assert(uris.length === sampleObjects.length); const metadatas = await Promise.all( - uris.map(async (m) => await storage.get(m)), + uris.map(async (m) => await storage.download(m)), ); for (let i = 0; i < metadatas.length; i++) { const expected = sampleObjects[i]; @@ -323,12 +318,10 @@ describe("IPFS Uploads", async () => { name: "TEST", }, ]; - const { baseUri: cid } = await storage.uploadBatch(sampleObjects); + const uris = await storage.uploadBatch(sampleObjects); assert( - (await getFile(`${cid}TEST`)).headers - ?.get("content-type") - ?.toString() === "image/jpeg", - `${cid}`, + (await getFile(uris[1])).headers?.get("content-type")?.toString() === + "image/jpeg", ); }); From b99740069c05d8da25933b776ece1fa9625de13f Mon Sep 17 00:00:00 2001 From: adam-maj Date: Thu, 15 Sep 2022 16:20:27 -0700 Subject: [PATCH 05/19] Update IPFS utils and test cases --- packages/storage/src/common/utils.ts | 66 ++- packages/storage/src/core/storage.ts | 58 +-- .../src/core/uploaders/ipfs-uploader.ts | 13 +- packages/storage/src/types/data.ts | 5 +- packages/storage/src/types/upload.ts | 7 +- packages/storage/test/ipfs.test.ts | 413 +++--------------- packages/storage/test/recursive.test.ts | 41 -- 7 files changed, 158 insertions(+), 445 deletions(-) delete mode 100644 packages/storage/test/recursive.test.ts diff --git a/packages/storage/src/common/utils.ts b/packages/storage/src/common/utils.ts index 553a2669c13..dac3bf44c4a 100644 --- a/packages/storage/src/common/utils.ts +++ b/packages/storage/src/common/utils.ts @@ -73,6 +73,11 @@ export function replaceObjectGatewayUrlsWithSchemes( return data; } + const { success } = FileOrBufferSchema.safeParse(data); + if (success) { + return data; + } + if (Array.isArray(data)) { return data.map((entry) => replaceObjectGatewayUrlsWithSchemes( @@ -82,12 +87,15 @@ export function replaceObjectGatewayUrlsWithSchemes( ); } - return Object.keys(data).map((key) => - replaceObjectGatewayUrlsWithSchemes( - data[key] as Exclude, - gatewayUrls, - ), + const json: Json = {}; + Object.keys(data).forEach( + (key) => + (json[key] = replaceObjectGatewayUrlsWithSchemes( + data[key] as Exclude, + gatewayUrls, + )), ); + return json; } return data; @@ -105,6 +113,11 @@ export function replaceObjectSchemesWithGatewayUrls( return data; } + const { success } = FileOrBufferSchema.safeParse(data); + if (success) { + return data; + } + if (Array.isArray(data)) { return data.map((entry) => replaceObjectSchemesWithGatewayUrls( @@ -114,12 +127,14 @@ export function replaceObjectSchemesWithGatewayUrls( ); } - return Object.keys(data).map((key) => - replaceObjectSchemesWithGatewayUrls( + const json: Json = {}; + Object.keys(data).forEach((key) => { + json[key] = replaceObjectSchemesWithGatewayUrls( data[key] as Exclude, gatewayUrls, - ), - ); + ); + }); + return json; } return data; @@ -143,22 +158,25 @@ export function extractObjectFiles( if (Array.isArray(data)) { data.forEach((entry) => extractObjectFiles(entry, files)); + } else { + Object.keys(data).map((key) => + extractObjectFiles((data as JsonObject)[key], files), + ); } - - Object.keys(data).map((key) => - extractObjectFiles((data as JsonObject)[key], files), - ); } return files; } export function replaceObjectFilesWithUris(data: Json, uris: string[]): Json { - // If item is a FileOrBuffer add it to our list of files const { success: isFileOrBuffer } = FileOrBufferSchema.safeParse(data); if (isFileOrBuffer) { - data = uris.splice(0, 1); - return data; + if (uris.length) { + data = uris.shift() as string; + return data; + } else { + console.warn("Not enough URIs to replace all files in object."); + } } if (typeof data === "object") { @@ -167,12 +185,18 @@ export function replaceObjectFilesWithUris(data: Json, uris: string[]): Json { } if (Array.isArray(data)) { - data.forEach((entry) => replaceObjectFilesWithUris(entry, uris)); + return data.map((entry) => replaceObjectFilesWithUris(entry, uris)); + } else { + const json: Json = {}; + Object.keys(data).map( + (key) => + (json[key] = replaceObjectFilesWithUris( + (data as JsonObject)[key], + uris, + )), + ); + return json; } - - Object.keys(data).map((key) => - replaceObjectFilesWithUris((data as JsonObject)[key], uris), - ); } return data; diff --git a/packages/storage/src/core/storage.ts b/packages/storage/src/core/storage.ts index bcea5be8d4d..5874020aad0 100644 --- a/packages/storage/src/core/storage.ts +++ b/packages/storage/src/core/storage.ts @@ -7,6 +7,7 @@ import { import { FileOrBuffer, FileOrBufferSchema, + IpfsUploadBatchOptions, IStorageDownloader, IStorageUploader, Json, @@ -17,49 +18,44 @@ import { import { StorageDownloader } from "./downloaders/storage-downloader"; import { IpfsUploader } from "./uploaders/ipfs-uploader"; -export class ThirdwebStorage { - private uploader: IStorageUploader; +export class ThirdwebStorage { + private uploader: IStorageUploader; private downloader: IStorageDownloader; constructor( - uploader: IStorageUploader = new IpfsUploader(), + uploader: IStorageUploader = new IpfsUploader(), downloader: IStorageDownloader = new StorageDownloader(), ) { this.uploader = uploader; this.downloader = downloader; } - async download(url: string): Promise { - const res = await this.downloader.download(url); - - const text = await res.text(); - - // Handle both JSON and standard text data types - try { - // If we get a JSON object, recursively replace any schemes with gatewayUrls - const json = JSON.parse(text); - return replaceObjectSchemesWithGatewayUrls( - json, - this.downloader.gatewayUrls, - ); - } catch { - return text; - } + async download(url: string): Promise { + return this.downloader.download(url); + } + + async downloadJSON(url: string): Promise { + const res = await this.download(url); + + // If we get a JSON object, recursively replace any schemes with gatewayUrls + const json = await res.json(); + return replaceObjectSchemesWithGatewayUrls( + json, + this.downloader.gatewayUrls, + ) as T; } - async upload( - data: JsonObject | FileOrBuffer, - options?: UploadOptions, - ): Promise { - const [uri] = await this.uploadBatch([data] as - | JsonObject[] - | FileOrBuffer[]); + async upload(data: JsonObject | FileOrBuffer, options?: T): Promise { + const [uri] = await this.uploadBatch( + [data] as JsonObject[] | FileOrBuffer[], + options, + ); return uri; } async uploadBatch( data: JsonObject[] | FileOrBuffer[], - options?: UploadOptions, + options?: T, ): Promise { const parsed: JsonObject[] | FileOrBuffer[] = UploadDataSchema.parse(data); const { success: isFileArray } = FileOrBufferSchema.safeParse(parsed[0]); @@ -81,11 +77,11 @@ export class ThirdwebStorage { data: JsonObject[], ): Promise { let cleaned = data; - if (this.downloader.gatewayUrls) { + if (this.uploader.gatewayUrls) { // Replace any gateway URLs with their hashes cleaned = replaceObjectGatewayUrlsWithSchemes( data, - this.downloader.gatewayUrls, + this.uploader.gatewayUrls, ) as JsonObject[]; } @@ -101,6 +97,10 @@ export class ThirdwebStorage { // Recurse through data and extract files to upload const files = extractObjectFiles(data); + if (!files.length) { + return data; + } + // Upload all files that came from the object const uris = await this.uploader.uploadBatch(files); diff --git a/packages/storage/src/core/uploaders/ipfs-uploader.ts b/packages/storage/src/core/uploaders/ipfs-uploader.ts index 8a32b229e7b..5f5ff4234ba 100644 --- a/packages/storage/src/core/uploaders/ipfs-uploader.ts +++ b/packages/storage/src/core/uploaders/ipfs-uploader.ts @@ -18,7 +18,7 @@ import { import fetch from "cross-fetch"; import FormData from "form-data"; -export class IpfsUploader implements IStorageUploader { +export class IpfsUploader implements IStorageUploader { public gatewayUrls: GatewayUrls; public uploadWithGatewayUrl: boolean; @@ -31,8 +31,8 @@ export class IpfsUploader implements IStorageUploader { data: (string | FileOrBuffer)[], options?: IpfsUploadBatchOptions, ): Promise { - const form = new FormData(); - const { fileNames } = this.buildFormData(form, data); + const formData = new FormData(); + const { form, fileNames } = this.buildFormData(formData, data); if (isBrowser()) { return this.uploadBatchBrowser(form, fileNames, options); @@ -50,6 +50,9 @@ export class IpfsUploader implements IStorageUploader { private async getUploadToken(): Promise { const res = await fetch(`${TW_IPFS_SERVER_URL}/grant`, { method: "GET", + headers: { + "X-APP-NAME": "Storage SDK", + }, }); if (!res.ok) { throw new Error(`Failed to get upload token`); @@ -115,6 +118,9 @@ export class IpfsUploader implements IStorageUploader { } }); + const metadata = { name: `Storage SDK`, keyvalues: {} }; + form.append("pinataMetadata", JSON.stringify(metadata)); + return { form, fileNames, @@ -185,6 +191,7 @@ export class IpfsUploader implements IStorageUploader { }); const body = await res.json(); if (!res.ok) { + console.warn(body); throw new Error("Failed to upload files to IPFS"); } diff --git a/packages/storage/src/types/data.ts b/packages/storage/src/types/data.ts index 084bb1b94ef..769fde1cd48 100644 --- a/packages/storage/src/types/data.ts +++ b/packages/storage/src/types/data.ts @@ -1,6 +1,9 @@ -import { isBrowser } from "../common/utils"; import { z } from "zod"; +function isBrowser() { + return typeof window !== "undefined"; +} + const JsonLiteralSchema = z.union([ z.string(), z.number(), diff --git a/packages/storage/src/types/upload.ts b/packages/storage/src/types/upload.ts index 72a60f05809..0714e244204 100644 --- a/packages/storage/src/types/upload.ts +++ b/packages/storage/src/types/upload.ts @@ -3,13 +3,10 @@ import { GatewayUrls } from "./download"; export type UploadOptions = { [key: string]: any }; -export interface IStorageUploader { +export interface IStorageUploader { gatewayUrls?: GatewayUrls; uploadWithGatewayUrl?: boolean; - uploadBatch( - data: (string | FileOrBuffer)[], - options?: UploadOptions, - ): Promise; + uploadBatch(data: (string | FileOrBuffer)[], options?: T): Promise; } export type UploadProgressEvent = { diff --git a/packages/storage/test/ipfs.test.ts b/packages/storage/test/ipfs.test.ts index 88a7aba097a..874bc2fd409 100644 --- a/packages/storage/test/ipfs.test.ts +++ b/packages/storage/test/ipfs.test.ts @@ -1,366 +1,89 @@ import { ThirdwebStorage } from "../src"; -import { DEFAULT_GATEWAY_URLS } from "../src/common/urls"; -import { - BufferOrStringWithName, - FileOrBuffer, - IpfsUploadBatchOptions, -} from "../src/types"; -import { assert, expect } from "chai"; -import fetch from "cross-fetch"; +import { assert } from "chai"; import { readFileSync } from "fs"; -const ipfsGatewayUrl = DEFAULT_GATEWAY_URLS["ipfs://"][0]; - -global.fetch = require("cross-fetch"); - -describe("IPFS Uploads", async () => { +describe("IPFS", async () => { const storage = new ThirdwebStorage(); - async function getFile(upload: string): Promise { - const response = await fetch( - `${ipfsGatewayUrl}${upload.replace("ipfs://", "")}`, - ) - .then(async (res) => { - return res; - }) - .catch((e) => { - assert.fail(e); - }); - - return response; - } - - describe("SDK storage interface", async () => { - it("Should single upload and fetch", async () => { - const uri = await storage.upload({ - name: "Test Name", - description: "Test Description", - }); - - const data = await storage.download(uri); - - expect(data.name).to.equal("Test Name"); - expect(data.description).to.equal("Test Description"); - }); - - it("Should batch upload and fetch", async () => { - const uris = await storage.uploadBatch([ - { - name: "Test Name 1", - description: "Test Description 1", - }, - { - name: "Test Name 2", - description: "Test Description 2", - }, - ]); - - const data1 = await storage.download(uris[0]); - const data2 = await storage.download(uris[1]); - - expect(data1.name).to.equal("Test Name 1"); - expect(data1.description).to.equal("Test Description 1"); - - expect(data2.name).to.equal("Test Name 2"); - expect(data2.description).to.equal("Test Description 2"); - }); + it("Should upload buffer with file number", async () => { + const uri = await storage.upload(readFileSync("test/files/0.jpg")); + assert(uri.endsWith("0")); }); - describe("Custom contract metadata", async () => { - it("should upload null metadata", async () => { - try { - const metadata = JSON.parse( - readFileSync("test/files/metadata.json", "utf8"), - ); - const uris = await storage.uploadBatch([metadata]); - assert.isTrue(uris.length > 0); - } catch (err) { - assert.fail(err as string); - } - }); - - it("should upload metadata and replace gateway urls on upload/download", async () => { - const upload = await storage.upload({ - svg: `${ipfsGatewayUrl}QmZsU8nTTexTxPzCKZKqo3Ntf5cUiWMRahoLmtpimeaCiT/backgrounds/SVG/Asset%20501.svg`, - }); - const downloaded = await storage.download(upload); - expect(downloaded.svg).to.eq( - `${DEFAULT_GATEWAY_URLS[1]}QmZsU8nTTexTxPzCKZKqo3Ntf5cUiWMRahoLmtpimeaCiT/backgrounds/SVG/Asset%20501.svg`, - ); + it("Should upload buffer with name", async () => { + const uri = await storage.upload({ + name: "0.jpg", + data: readFileSync("test/files/0.jpg"), }); + assert(uri.endsWith("0.jpg")); + }); - it.skip("should expose upload progress", async () => { - let updates = 0; - await storage.uploadBatch( - [ - readFileSync("test/files/0.jpg"), - readFileSync("test/files/1.jpg"), - readFileSync("test/files/test.mp4"), - ], + it("Should upload and download JSON object", async () => { + const uri = await storage.upload({ + name: "Goku", + description: "The strongest human in the world", + properties: [ { - onProgress: () => { - updates += 1; - }, - }, - ); - - expect(updates).to.be.greaterThan(0); - }); - - it("should upload a file through any property, even when it is in an object nested inside another object", async () => { - try { - const upload = await storage.upload({ - name: "test", - image: readFileSync("test/files/bone.jpg"), - test: { - test: { - image: readFileSync("test/files/bone.jpg"), - }, - }, - }); - const data = await (await getFile(upload)).json(); - const uploadTest = (await getFile(data.test.test.image)).headers - ?.get("content-type") - ?.toString(); - - assert.equal(uploadTest, "image/jpeg"); - } catch (err) { - assert.fail(err as string); - } - }); - - it("should not upload the string to IPFS", async () => { - const upload = await storage.upload({ - image: - "ipfs://QmZsU8nTTexTxPzCKZKqo3Ntf5cUiWMRahoLmtpimeaCiT/face_parts/Asset%20331.svg", - }); - assert.equal( - upload, - "ipfs://QmYKJLPfwKduSfWgdLLt49SE6LvzkGzxeYMCkhXWbpJam7/0", - ); - }); - - it("should upload an MP4 file when passed in the animation_url property", async () => { - const upload = await storage.upload({ - animation_url: readFileSync("test/files/test.mp4"), - }); - assert.equal( - upload, - "ipfs://QmbaNzUcv7KPgdwq9u2qegcptktpUK6CdRZF72eSjSa6iJ/0", - ); - }); - - it("should upload an media file and resolve to gateway URL when fetching it", async () => { - const upload = await storage.upload({ - animation_url: readFileSync("test/files/test.mp4"), - }); - assert.equal( - upload, - "ipfs://QmbaNzUcv7KPgdwq9u2qegcptktpUK6CdRZF72eSjSa6iJ/0", - ); - const meta = await storage.download(upload); - assert.equal( - meta.animation_url, - `${ipfsGatewayUrl}QmUphf8LnNGdFwBevnxNkq8dxcZ4qxzzPjoNMDkSQfECKM/0`, - ); - }); - - it("should upload many objects correctly", async () => { - const sampleObjects: { id: number; description: string; prop: string }[] = - [ - { - id: 0, - description: "test 0", - prop: "123", - }, - { - id: 1, - description: "test 1", - prop: "321", - }, - ]; - const serialized = sampleObjects.map((o) => - Buffer.from(JSON.stringify(o)), - ); - const uris = await storage.uploadBatch(serialized, { - rewriteFileNames: { - fileStartNumber: 1, + name: "Strength", + value: "100", }, - } as IpfsUploadBatchOptions); - for (const object of sampleObjects) { - assert(uris[object.id].endsWith(object.id.toString())); - const parsed = await storage.download(uris[object.id]); - assert.equal(parsed.description, object.description); - assert.equal(parsed.id, object.id); - } + ], }); + assert(uri.endsWith("0")); - it("should upload files with filenames correctly", async () => { - const sampleObjects: BufferOrStringWithName[] = [ - { - data: readFileSync("test/files/test.mp4"), - name: "test2.mp4", - }, - { data: readFileSync("test/files/test.mp4"), name: "test3.mp4" }, - { - data: readFileSync("test/files/bone.jpg"), - name: "test.jpeg", - }, - ]; - const uris = await storage.uploadBatch(sampleObjects); - assert( - (await getFile(uris[2])).headers?.get("content-type")?.toString() === - "image/jpeg", - ); - }); + const data = await storage.downloadJSON(uri); + assert(data.name === "Goku"); + assert(data.description === "The strongest human in the world"); + assert(data.properties.length === 1); + }); - it("should upload files according to passed start file number", async () => { - const sampleObjects: FileOrBuffer[] = [ - readFileSync("test/files/test.mp4"), - readFileSync("test/files/bone.jpg"), - ]; - const uris = await storage.uploadBatch(sampleObjects, { - rewriteFileNames: { - fileStartNumber: 1, - }, - } as IpfsUploadBatchOptions); - assert( - (await getFile(uris[1])).headers?.get("content-type")?.toString() === - "image/jpeg", - ); - }); - it("should upload files according to start file number as 0", async () => { - const sampleObjects = [ - readFileSync("test/files/bone.jpg"), - readFileSync("test/files/test.mp4"), - ]; - const uris = await storage.uploadBatch(sampleObjects); - assert( - (await getFile(uris[0])).headers?.get("content-type")?.toString() === - "image/jpeg", - ); - }); - it("should upload properties recursively in batch", async () => { - const sampleObjects: any[] = [ - { - name: "test 0", - image: readFileSync("test/files/test.mp4"), - }, - { - name: "test 1", - image: readFileSync("test/files/1.jpg"), - properties: { - image: readFileSync("test/files/2.jpg"), - }, - }, - { - name: "test 2", - image: readFileSync("test/files/3.jpg"), - properties: { - image: readFileSync("test/files/4.jpg"), - test: { - image: readFileSync("test/files/5.jpg"), - }, - }, - }, - ]; - const uris = await storage.uploadBatch(sampleObjects); - assert(uris.length === sampleObjects.length); - const [metadata1, metadata2, metadata3] = await Promise.all( - ( - await Promise.all(uris.map((m) => getFile(m))) - ).map((m: any) => m.json()), - ); - assert( - metadata1.image === - "ipfs://QmTpv5cWy677mgABsgJgwZ6pe2bEpSWQTvcCb8Hmj3ac8E/0", - ); - assert( - metadata2.image === - "ipfs://QmTpv5cWy677mgABsgJgwZ6pe2bEpSWQTvcCb8Hmj3ac8E/1", - ); - assert( - metadata3.image === - "ipfs://QmTpv5cWy677mgABsgJgwZ6pe2bEpSWQTvcCb8Hmj3ac8E/3", - ); - }); + it("Should batch upload buffers with names", async () => { + const uris = await storage.uploadBatch([ + { + data: readFileSync("test/files/0.jpg"), + name: "0.jpg", + }, + { + data: readFileSync("test/files/1.jpg"), + name: "1.jpg", + }, + ]); + + assert(uris[0].endsWith("0.jpg")); + assert(uris[1].endsWith("1.jpg")); + }); - it("should upload properties in right order", async () => { - const sampleObjects: any[] = []; - for (let i = 0; i < 30; i++) { - const nft = { - name: `${i}`, - image: readFileSync(`test/files/${i % 5}.jpg`), - }; - sampleObjects.push(nft); - } - const uris = await storage.uploadBatch(sampleObjects); - assert(uris.length === sampleObjects.length); - const metadatas = await Promise.all( - uris.map(async (m) => await storage.download(m)), - ); - for (let i = 0; i < metadatas.length; i++) { - const expected = sampleObjects[i]; - const downloaded = metadatas[i]; - expect(downloaded.name).to.be.eq(expected.name); - expect(downloaded.image.endsWith(`${i}`)).to.eq(true); - } - }); + it("Should batch upload JSON objects", async () => { + const uris = await storage.uploadBatch([ + { + name: "Goku", + strength: 100, + powerLevel: "Over 9000", + }, + { + name: "Vegeta", + strenth: 90, + powerLevel: "5000", + }, + ]); + + assert(uris[0].endsWith("0")); + assert(uris[1].endsWith("1")); + + const data0 = await storage.downloadJSON(uris[0]); + const data1 = await storage.downloadJSON(uris[1]); + + assert(data0.name === "Goku"); + assert(data1.name === "Vegeta"); + }); - it("should upload properly with same file names but one with capitalized letters", async () => { - const sampleObjects: BufferOrStringWithName[] = [ - { - data: readFileSync("test/files/test.mp4"), - name: "test", - }, - { - data: readFileSync("test/files/bone.jpg"), - name: "TEST", - }, - ]; - const uris = await storage.uploadBatch(sampleObjects); - assert( - (await getFile(uris[1])).headers?.get("content-type")?.toString() === - "image/jpeg", - ); + it("Should upload an MP4 file", async () => { + const uri = await storage.upload({ + animation_url: readFileSync("test/files/test.mp4"), }); - it("should throw an error when trying to upload two files with the same name", async () => { - const sampleObjects: BufferOrStringWithName[] = [ - { - data: readFileSync("test/files/test.mp4"), - name: "test", - }, - { - data: readFileSync("test/files/bone.jpg"), - name: "test", - }, - ]; - try { - await storage.uploadBatch(sampleObjects); - assert.fail("should throw an error"); - } catch (e) { - assert.equal(true, true); - } - }); + console.log(uri); - it("bulk upload", async () => { - const sampleObjects: BufferOrStringWithName[] = [ - { - data: readFileSync("test/files/test.mp4"), - name: "test", - }, - { - data: readFileSync("test/files/bone.jpg"), - name: "test", - }, - ]; - try { - await storage.uploadBatch(sampleObjects); - assert.fail("should throw an error"); - } catch (e) { - assert.equal(true, true); - } - }); + assert(uri === "ipfs://QmbaNzUcv7KPgdwq9u2qegcptktpUK6CdRZF72eSjSa6iJ/0"); }); }); diff --git a/packages/storage/test/recursive.test.ts b/packages/storage/test/recursive.test.ts deleted file mode 100644 index 5f3fc9e0e2d..00000000000 --- a/packages/storage/test/recursive.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { IpfsStorage } from "../src/core/ipfs-storage"; -import { assert } from "chai"; - -const ipfsGatewayUrl = "https://ipfs.thirdweb.com/ipfs/"; -const storage = new IpfsStorage(ipfsGatewayUrl); - -describe("Recursive Testing", async () => { - let json: Record; - beforeEach(async () => { - json = { - test: "test", - test2: "ipfs://QmTJyQwBhbELaceUScbM29G3HRpk4GdKXMuVxAxGCGtXME", - test3: { - test: "test", - test2: "ipfs://QmTJyQwBhbELaceUScbM29G3HRpk4GdKXMuVxAxGCGtXME", - test3: { - test: "test", - test2: "ipfs://QmTJyQwBhbELaceUScbM29G3HRpk4GdKXMuVxAxGCGtXME", - }, - }, - }; - }); - it("should resolve all URLs", async () => { - const uri = await storage.uploadMetadata(json); - const jsonOutput = await storage.get(uri); - assert.notStrictEqual(jsonOutput, { - test2: - "https://ipfs.thirdweb.com/ipfs/QmTJyQwBhbELaceUScbM29G3HRpk4GdKXMuVxAxGCGtXME", - test3: { - test: "test", - test2: - "https://ipfs.thirdweb.com/ipfs/QmTJyQwBhbELaceUScbM29G3HRpk4GdKXMuVxAxGCGtXME", - test3: { - test: "test", - test2: - "https://ipfs.thirdweb.com/ipfs/QmTJyQwBhbELaceUScbM29G3HRpk4GdKXMuVxAxGCGtXME", - }, - }, - }); - }); -}); From 2cce89f46b5c3b39f7dfacba722421e4972c1a7a Mon Sep 17 00:00:00 2001 From: adam-maj Date: Fri, 16 Sep 2022 13:57:07 -0700 Subject: [PATCH 06/19] All test cases passing --- packages/storage/package.json | 2 +- packages/storage/src/common/urls.ts | 10 +- packages/storage/src/core/storage.ts | 18 +- .../src/core/uploaders/ipfs-uploader.ts | 5 +- packages/storage/src/types/upload.ts | 3 +- packages/storage/test/ipfs.test.ts | 195 ++++++++++++++++-- 6 files changed, 201 insertions(+), 32 deletions(-) diff --git a/packages/storage/package.json b/packages/storage/package.json index f682e9594e7..1c36a06aca5 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -34,4 +34,4 @@ "form-data": "^4.0.0", "zod": "^3.11.6" } -} +} \ No newline at end of file diff --git a/packages/storage/src/common/urls.ts b/packages/storage/src/common/urls.ts index 098c19df798..17d2d5ea34c 100644 --- a/packages/storage/src/common/urls.ts +++ b/packages/storage/src/common/urls.ts @@ -4,6 +4,7 @@ import { GatewayUrls } from "../types"; * @internal */ export const DEFAULT_GATEWAY_URLS: GatewayUrls = { + // Note: Gateway URLs should have trailing slashes (we clean this on user input) "ipfs://": [ "https://gateway.ipfscdn.io/ipfs/", "https://cloudflare-ipfs.com/ipfs/", @@ -29,7 +30,14 @@ export function prepareGatewayUrls(gatewayUrls?: GatewayUrls): GatewayUrls { for (const key of Object.keys(DEFAULT_GATEWAY_URLS)) { if (gatewayUrls && gatewayUrls[key]) { - allGatewayUrls[key] = [...gatewayUrls[key], ...DEFAULT_GATEWAY_URLS[key]]; + // Make sure that all user gateway URLs have trailing slashes + const cleanedGatewayUrls = gatewayUrls[key].map( + (url) => url.replace(/\/$/, "") + "/", + ); + allGatewayUrls[key] = [ + ...cleanedGatewayUrls, + ...DEFAULT_GATEWAY_URLS[key], + ]; } } diff --git a/packages/storage/src/core/storage.ts b/packages/storage/src/core/storage.ts index 5874020aad0..c969ab95de7 100644 --- a/packages/storage/src/core/storage.ts +++ b/packages/storage/src/core/storage.ts @@ -67,7 +67,10 @@ export class ThirdwebStorage { // Otherwise it is an array of JSON objects, so we have to prepare it first const metadata = ( - await this.uploadAndReplaceFilesWithHashes(parsed as JsonObject[]) + await this.uploadAndReplaceFilesWithHashes( + parsed as JsonObject[], + options, + ) ).map((item) => JSON.stringify(item)); return this.uploader.uploadBatch(metadata, options); @@ -75,36 +78,37 @@ export class ThirdwebStorage { private async uploadAndReplaceFilesWithHashes( data: JsonObject[], + options?: T, ): Promise { let cleaned = data; if (this.uploader.gatewayUrls) { // Replace any gateway URLs with their hashes cleaned = replaceObjectGatewayUrlsWithSchemes( - data, + cleaned, this.uploader.gatewayUrls, ) as JsonObject[]; } - if (this.uploader.uploadWithGatewayUrl) { + if (options?.uploadWithGatewayUrl) { // If flag is set, replace all schemes with their preferred gateway URL // Ex: used for Solana, where services don't resolve schemes for you, so URLs must be useable by default cleaned = replaceObjectSchemesWithGatewayUrls( - data, + cleaned, this.uploader.gatewayUrls || this.downloader.gatewayUrls, ) as JsonObject[]; } // Recurse through data and extract files to upload - const files = extractObjectFiles(data); + const files = extractObjectFiles(cleaned); if (!files.length) { - return data; + return cleaned; } // Upload all files that came from the object const uris = await this.uploader.uploadBatch(files); // Recurse through data and replace files with hashes - return replaceObjectFilesWithUris(data, uris) as JsonObject[]; + return replaceObjectFilesWithUris(cleaned, uris) as JsonObject[]; } } diff --git a/packages/storage/src/core/uploaders/ipfs-uploader.ts b/packages/storage/src/core/uploaders/ipfs-uploader.ts index 5f5ff4234ba..aa53cd00a69 100644 --- a/packages/storage/src/core/uploaders/ipfs-uploader.ts +++ b/packages/storage/src/core/uploaders/ipfs-uploader.ts @@ -20,11 +20,9 @@ import FormData from "form-data"; export class IpfsUploader implements IStorageUploader { public gatewayUrls: GatewayUrls; - public uploadWithGatewayUrl: boolean; constructor(options?: IpfsUploaderOptions) { this.gatewayUrls = prepareGatewayUrls(options?.gatewayUrls); - this.uploadWithGatewayUrl = options?.uploadWithGatewayUrl || false; } async uploadBatch( @@ -32,7 +30,7 @@ export class IpfsUploader implements IStorageUploader { options?: IpfsUploadBatchOptions, ): Promise { const formData = new FormData(); - const { form, fileNames } = this.buildFormData(formData, data); + const { form, fileNames } = this.buildFormData(formData, data, options); if (isBrowser()) { return this.uploadBatchBrowser(form, fileNames, options); @@ -100,7 +98,6 @@ export class IpfsUploader implements IStorageUploader { fileName = `${i}`; } } - const filepath = `files/${fileName}`; if (fileNames.indexOf(fileName) > -1) { throw new Error( diff --git a/packages/storage/src/types/upload.ts b/packages/storage/src/types/upload.ts index 0714e244204..af7f7414ea2 100644 --- a/packages/storage/src/types/upload.ts +++ b/packages/storage/src/types/upload.ts @@ -5,7 +5,6 @@ export type UploadOptions = { [key: string]: any }; export interface IStorageUploader { gatewayUrls?: GatewayUrls; - uploadWithGatewayUrl?: boolean; uploadBatch(data: (string | FileOrBuffer)[], options?: T): Promise; } @@ -23,12 +22,12 @@ export type UploadProgressEvent = { export type IpfsUploaderOptions = { gatewayUrls?: GatewayUrls; - uploadWithGatewayUrl?: boolean; }; export type IpfsUploadBatchOptions = { rewriteFileNames?: { fileStartNumber: number; }; + uploadWithGatewayUrl?: boolean; onProgress?: (event: UploadProgressEvent) => void; }; diff --git a/packages/storage/test/ipfs.test.ts b/packages/storage/test/ipfs.test.ts index 874bc2fd409..c22593bd00b 100644 --- a/packages/storage/test/ipfs.test.ts +++ b/packages/storage/test/ipfs.test.ts @@ -1,5 +1,6 @@ import { ThirdwebStorage } from "../src"; -import { assert } from "chai"; +import { DEFAULT_GATEWAY_URLS } from "../src/common/urls"; +import { expect } from "chai"; import { readFileSync } from "fs"; describe("IPFS", async () => { @@ -7,7 +8,8 @@ describe("IPFS", async () => { it("Should upload buffer with file number", async () => { const uri = await storage.upload(readFileSync("test/files/0.jpg")); - assert(uri.endsWith("0")); + + expect(uri.endsWith("0"), `${uri} does not end with '0'`).to.be.true; }); it("Should upload buffer with name", async () => { @@ -15,7 +17,8 @@ describe("IPFS", async () => { name: "0.jpg", data: readFileSync("test/files/0.jpg"), }); - assert(uri.endsWith("0.jpg")); + expect(uri.endsWith("0.jpg"), `${uri} does not end with '0.jpg'`).to.be + .true; }); it("Should upload and download JSON object", async () => { @@ -29,28 +32,84 @@ describe("IPFS", async () => { }, ], }); - assert(uri.endsWith("0")); + expect(uri.endsWith("0"), `${uri} does not end with '0'`).to.be.true; const data = await storage.downloadJSON(uri); - assert(data.name === "Goku"); - assert(data.description === "The strongest human in the world"); - assert(data.properties.length === 1); + expect(data.name).to.equal("Goku"); + expect(data.description).to.equal("The strongest human in the world"); + expect(data.properties.length).to.equal(1); }); it("Should batch upload buffers with names", async () => { const uris = await storage.uploadBatch([ { data: readFileSync("test/files/0.jpg"), - name: "0.jpg", + name: "first.jpg", }, { data: readFileSync("test/files/1.jpg"), - name: "1.jpg", + name: "second.jpg", }, ]); - assert(uris[0].endsWith("0.jpg")); - assert(uris[1].endsWith("1.jpg")); + expect( + uris[0].endsWith("first.jpg"), + `${uris[0]} does not end with 'first.jpg'`, + ).to.be.true; + expect( + uris[1].endsWith("second.jpg"), + `${uris[1]} does not end with '0.jpg'`, + ).to.be.true; + }); + + it("Should rewrite file names to numbers if specified", async () => { + const uris = await storage.uploadBatch( + [ + { + data: readFileSync("test/files/0.jpg"), + name: "first.jpg", + }, + { + data: readFileSync("test/files/1.jpg"), + name: "second.jpg", + }, + ], + { + rewriteFileNames: { + fileStartNumber: 0, + }, + }, + ); + + expect(uris[0].endsWith("0"), `${uris[0]} does not end with '0'`).to.be + .true; + expect(uris[1].endsWith("1"), `${uris[1]} does not end with '1'`).to.be + .true; + }); + + it("Should rewrite files with non zero start file number", async () => { + const uris = await storage.uploadBatch( + [ + { + data: readFileSync("test/files/0.jpg"), + name: "first.jpg", + }, + { + data: readFileSync("test/files/1.jpg"), + name: "second.jpg", + }, + ], + { + rewriteFileNames: { + fileStartNumber: 5, + }, + }, + ); + + expect(uris[0].endsWith("5"), `${uris[0]} does not end with '5'`).to.be + .true; + expect(uris[1].endsWith("6"), `${uris[1]} does not end with '6'`).to.be + .true; }); it("Should batch upload JSON objects", async () => { @@ -67,14 +126,16 @@ describe("IPFS", async () => { }, ]); - assert(uris[0].endsWith("0")); - assert(uris[1].endsWith("1")); + expect(uris[0].endsWith("0"), `${uris[0]} does not end with '0'`).to.be + .true; + expect(uris[1].endsWith("1"), `${uris[1]} does not end with '0.jpg'`).to.be + .true; const data0 = await storage.downloadJSON(uris[0]); const data1 = await storage.downloadJSON(uris[1]); - assert(data0.name === "Goku"); - assert(data1.name === "Vegeta"); + expect(data0.name).to.equal("Goku"); + expect(data1.name).to.equal("Vegeta"); }); it("Should upload an MP4 file", async () => { @@ -82,8 +143,108 @@ describe("IPFS", async () => { animation_url: readFileSync("test/files/test.mp4"), }); - console.log(uri); + expect(uri).to.equal( + "ipfs://QmbaNzUcv7KPgdwq9u2qegcptktpUK6CdRZF72eSjSa6iJ/0", + ); + }); + + it("Should replace gateway URLs with schemes on upload", async () => { + const uri = await storage.upload({ + image: `${DEFAULT_GATEWAY_URLS["ipfs://"][0]}QmbaNzUcv7KPgdwq9u2qegcptktpUK6CdRZF72eSjSa6iJ/0`, + }); + + const res = await storage.download(uri); + const json = await res.json(); + + expect(json.image).to.equal( + "ipfs://QmbaNzUcv7KPgdwq9u2qegcptktpUK6CdRZF72eSjSa6iJ/0", + ); + }); + + it("Should replace schemes with gateway URLs on download", async () => { + const uri = await storage.upload({ + image: "ipfs://QmbaNzUcv7KPgdwq9u2qegcptktpUK6CdRZF72eSjSa6iJ/0", + }); + + const json = await storage.downloadJSON(uri); + + expect(json.image).to.equal( + `${DEFAULT_GATEWAY_URLS["ipfs://"][0]}QmbaNzUcv7KPgdwq9u2qegcptktpUK6CdRZF72eSjSa6iJ/0`, + ); + }); + + it("Should upload files with gateway URLs if specified", async () => { + const uri = await storage.upload( + { + // Gateway URLs should first be converted back to ipfs:// and then all ipfs:// should convert to first gateway URL + image: `${DEFAULT_GATEWAY_URLS["ipfs://"][1]}QmbaNzUcv7KPgdwq9u2qegcptktpUK6CdRZF72eSjSa6iJ/0`, + animation_url: + "ipfs://QmbaNzUcv7KPgdwq9u2qegcptktpUK6CdRZF72eSjSa6iJ/0", + }, + { + uploadWithGatewayUrl: true, + }, + ); + + const res = await storage.download(uri); + const json = await res.json(); + + expect(json.image).to.equal( + `${DEFAULT_GATEWAY_URLS["ipfs://"][0]}QmbaNzUcv7KPgdwq9u2qegcptktpUK6CdRZF72eSjSa6iJ/0`, + ); + expect(json.animation_url).to.equal( + `${DEFAULT_GATEWAY_URLS["ipfs://"][0]}QmbaNzUcv7KPgdwq9u2qegcptktpUK6CdRZF72eSjSa6iJ/0`, + ); + }); + + it("Should throw an error when trying to upload files with the same name", async () => { + try { + await storage.uploadBatch([ + { + data: readFileSync("test/files/0.jpg"), + name: "0.jpg", + }, + { + data: readFileSync("test/files/1.jpg"), + name: "0.jpg", + }, + ]); + expect.fail("Uploading files with same name did not throw an error."); + } catch (err: any) { + expect(err.message).to.contain( + "[DUPLICATE_FILE_NAME_ERROR] File name 0.jpg was passed for more than one file.", + ); + } + }); + + it("Should recursively upload and replace files", async () => { + // Should test nested within objects and arrays + const uris = await storage.uploadBatch([ + { + image: readFileSync("test/files/0.jpg"), + properties: [ + { + image: readFileSync("test/files/1.jpg"), + }, + { + animation_url: readFileSync("test/files/2.jpg"), + }, + ], + }, + { + image: readFileSync("test/files/3.jpg"), + properties: [ + readFileSync("test/files/4.jpg"), + readFileSync("test/files/test.mp4"), + ], + }, + ]); - assert(uri === "ipfs://QmbaNzUcv7KPgdwq9u2qegcptktpUK6CdRZF72eSjSa6iJ/0"); + expect(uris[0]).to.equal( + "ipfs://QmTtEY2WSTDpzYSXw2G3xsYw3eMs8YephvrfVYd8qia9F9/0", + ); + expect(uris[1]).to.equal( + "ipfs://QmTtEY2WSTDpzYSXw2G3xsYw3eMs8YephvrfVYd8qia9F9/1", + ); }); }); From 0c12cad55055902198f89160bb5cd1c68e3f5066 Mon Sep 17 00:00:00 2001 From: adam-maj Date: Fri, 16 Sep 2022 14:00:48 -0700 Subject: [PATCH 07/19] Update style --- packages/storage/src/core/storage.ts | 5 ++--- packages/storage/test/ipfs.test.ts | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/storage/src/core/storage.ts b/packages/storage/src/core/storage.ts index c969ab95de7..acdb8305358 100644 --- a/packages/storage/src/core/storage.ts +++ b/packages/storage/src/core/storage.ts @@ -10,7 +10,6 @@ import { IpfsUploadBatchOptions, IStorageDownloader, IStorageUploader, - Json, JsonObject, UploadDataSchema, UploadOptions, @@ -34,7 +33,7 @@ export class ThirdwebStorage { return this.downloader.download(url); } - async downloadJSON(url: string): Promise { + async downloadJSON(url: string): Promise { const res = await this.download(url); // If we get a JSON object, recursively replace any schemes with gatewayUrls @@ -42,7 +41,7 @@ export class ThirdwebStorage { return replaceObjectSchemesWithGatewayUrls( json, this.downloader.gatewayUrls, - ) as T; + ) as TJSON; } async upload(data: JsonObject | FileOrBuffer, options?: T): Promise { diff --git a/packages/storage/test/ipfs.test.ts b/packages/storage/test/ipfs.test.ts index c22593bd00b..30695b16e98 100644 --- a/packages/storage/test/ipfs.test.ts +++ b/packages/storage/test/ipfs.test.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-unused-expressions */ import { ThirdwebStorage } from "../src"; import { DEFAULT_GATEWAY_URLS } from "../src/common/urls"; import { expect } from "chai"; From d9d2cc97671cc59898c241339906a25d5b20675e Mon Sep 17 00:00:00 2001 From: adam-maj Date: Fri, 16 Sep 2022 14:37:33 -0700 Subject: [PATCH 08/19] Add uploadWithoutDirectory option to upload --- .../storage/src/core/downloaders/index.ts | 1 + packages/storage/src/core/index.ts | 2 + packages/storage/src/core/uploaders/index.ts | 1 + .../src/core/uploaders/ipfs-uploader.ts | 34 ++++++++++++++-- packages/storage/src/types/upload.ts | 1 + packages/storage/test/ipfs.test.ts | 39 +++++++++++++++++++ 6 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 packages/storage/src/core/downloaders/index.ts create mode 100644 packages/storage/src/core/uploaders/index.ts diff --git a/packages/storage/src/core/downloaders/index.ts b/packages/storage/src/core/downloaders/index.ts new file mode 100644 index 00000000000..d61849d80f8 --- /dev/null +++ b/packages/storage/src/core/downloaders/index.ts @@ -0,0 +1 @@ +export { StorageDownloader } from "./storage-downloader"; diff --git a/packages/storage/src/core/index.ts b/packages/storage/src/core/index.ts index 433c831b99a..d2f51d39c27 100644 --- a/packages/storage/src/core/index.ts +++ b/packages/storage/src/core/index.ts @@ -1 +1,3 @@ export { ThirdwebStorage } from "./storage"; +export * from "./downloaders"; +export * from "./uploaders"; diff --git a/packages/storage/src/core/uploaders/index.ts b/packages/storage/src/core/uploaders/index.ts new file mode 100644 index 00000000000..61c17cd46cb --- /dev/null +++ b/packages/storage/src/core/uploaders/index.ts @@ -0,0 +1 @@ +export { IpfsUploader } from "./ipfs-uploader"; diff --git a/packages/storage/src/core/uploaders/ipfs-uploader.ts b/packages/storage/src/core/uploaders/ipfs-uploader.ts index aa53cd00a69..65675e49bea 100644 --- a/packages/storage/src/core/uploaders/ipfs-uploader.ts +++ b/packages/storage/src/core/uploaders/ipfs-uploader.ts @@ -29,6 +29,12 @@ export class IpfsUploader implements IStorageUploader { data: (string | FileOrBuffer)[], options?: IpfsUploadBatchOptions, ): Promise { + if (options?.uploadWithoutDirectory && data.length > 1) { + throw new Error( + "[UPLOAD_WITHOUT_DIRECTORY_ERROR] Cannot upload more than one file or object without directory!", + ); + } + const formData = new FormData(); const { form, fileNames } = this.buildFormData(formData, data, options); @@ -98,7 +104,12 @@ export class IpfsUploader implements IStorageUploader { fileName = `${i}`; } } - const filepath = `files/${fileName}`; + + // If we don't want to wrap with directory, adjust the filepath + const filepath = options?.uploadWithoutDirectory + ? `files` + : `files/${fileName}`; + if (fileNames.indexOf(fileName) > -1) { throw new Error( `[DUPLICATE_FILE_NAME_ERROR] File name ${fileName} was passed for more than one file.`, @@ -118,6 +129,15 @@ export class IpfsUploader implements IStorageUploader { const metadata = { name: `Storage SDK`, keyvalues: {} }; form.append("pinataMetadata", JSON.stringify(metadata)); + if (options?.uploadWithoutDirectory) { + form.append( + "pinataOptions", + JSON.stringify({ + wrapWithDirectory: false, + }), + ); + } + return { form, fileNames, @@ -146,7 +166,11 @@ export class IpfsUploader implements IStorageUploader { throw new Error("Failed to upload files to IPFS"); } - resolve(fileNames.map((name) => `ipfs://${cid}/${name}`)); + if (options?.uploadWithoutDirectory) { + resolve([`ipfs://${cid}`]); + } else { + resolve(fileNames.map((name) => `ipfs://${cid}/${name}`)); + } }; xhr.onerror = (err) => { @@ -197,6 +221,10 @@ export class IpfsUploader implements IStorageUploader { throw new Error("Failed to upload files to IPFS"); } - return fileNames.map((name) => `ipfs://${cid}/${name}`); + if (options?.uploadWithoutDirectory) { + return [`ipfs://${cid}`]; + } else { + return fileNames.map((name) => `ipfs://${cid}/${name}`); + } } } diff --git a/packages/storage/src/types/upload.ts b/packages/storage/src/types/upload.ts index af7f7414ea2..22e987edda3 100644 --- a/packages/storage/src/types/upload.ts +++ b/packages/storage/src/types/upload.ts @@ -30,4 +30,5 @@ export type IpfsUploadBatchOptions = { }; uploadWithGatewayUrl?: boolean; onProgress?: (event: UploadProgressEvent) => void; + uploadWithoutDirectory?: boolean; }; diff --git a/packages/storage/test/ipfs.test.ts b/packages/storage/test/ipfs.test.ts index 30695b16e98..66705e37ffc 100644 --- a/packages/storage/test/ipfs.test.ts +++ b/packages/storage/test/ipfs.test.ts @@ -149,6 +149,45 @@ describe("IPFS", async () => { ); }); + it("Should upload without directory if specified", async () => { + const uri = await storage.upload( + { + name: "Upload Without Directory", + description: "Uploading alone without a directory...", + }, + { + uploadWithoutDirectory: true, + }, + ); + + expect(uri).to.equal( + "ipfs://QmdnBEP9UFcRfbuAyXFefNccNbuKWTscHrpWZatvqz9VcV", + ); + + const json = await storage.downloadJSON(uri); + + expect(json.name).to.equal("Upload Without Directory"); + expect(json.description).to.equal("Uploading alone without a directory..."); + }); + + it("Should throw an error on upload without directory with multiple uploads", async () => { + try { + await storage.uploadBatch( + [readFileSync("test/files/0.jpg"), readFileSync("test/files/1.jpg")], + { + uploadWithoutDirectory: true, + }, + ); + expect.fail( + "Failed to throw an error on uploading multiple files without directory", + ); + } catch (err: any) { + expect(err.message).to.contain( + "[UPLOAD_WITHOUT_DIRECTORY_ERROR] Cannot upload more than one file or object without directory!", + ); + } + }); + it("Should replace gateway URLs with schemes on upload", async () => { const uri = await storage.upload({ image: `${DEFAULT_GATEWAY_URLS["ipfs://"][0]}QmbaNzUcv7KPgdwq9u2qegcptktpUK6CdRZF72eSjSa6iJ/0`, From 35ddfcf67beb1de173541ce676c607440965d7db Mon Sep 17 00:00:00 2001 From: adam-maj Date: Fri, 16 Sep 2022 14:56:25 -0700 Subject: [PATCH 09/19] Add ability to set solana upload with gateway in constructor --- packages/storage/src/core/storage.ts | 21 +++++++------- .../src/core/uploaders/ipfs-uploader.ts | 2 ++ packages/storage/src/types/upload.ts | 2 ++ packages/storage/test/ipfs.test.ts | 29 +++++++++++++++++-- 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/packages/storage/src/core/storage.ts b/packages/storage/src/core/storage.ts index acdb8305358..19b0ab327a6 100644 --- a/packages/storage/src/core/storage.ts +++ b/packages/storage/src/core/storage.ts @@ -80,21 +80,22 @@ export class ThirdwebStorage { options?: T, ): Promise { let cleaned = data; - if (this.uploader.gatewayUrls) { + // TODO: Gateway URLs should probably be top-level since both uploader and downloader need them + if (this.uploader.gatewayUrls || this.downloader.gatewayUrls) { // Replace any gateway URLs with their hashes cleaned = replaceObjectGatewayUrlsWithSchemes( - cleaned, - this.uploader.gatewayUrls, - ) as JsonObject[]; - } - - if (options?.uploadWithGatewayUrl) { - // If flag is set, replace all schemes with their preferred gateway URL - // Ex: used for Solana, where services don't resolve schemes for you, so URLs must be useable by default - cleaned = replaceObjectSchemesWithGatewayUrls( cleaned, this.uploader.gatewayUrls || this.downloader.gatewayUrls, ) as JsonObject[]; + + if (options?.uploadWithGatewayUrl || this.uploader.uploadWithGatewayUrl) { + // If flag is set, replace all schemes with their preferred gateway URL + // Ex: used for Solana, where services don't resolve schemes for you, so URLs must be useable by default + cleaned = replaceObjectSchemesWithGatewayUrls( + cleaned, + this.uploader.gatewayUrls || this.downloader.gatewayUrls, + ) as JsonObject[]; + } } // Recurse through data and extract files to upload diff --git a/packages/storage/src/core/uploaders/ipfs-uploader.ts b/packages/storage/src/core/uploaders/ipfs-uploader.ts index 65675e49bea..a0e1be05fee 100644 --- a/packages/storage/src/core/uploaders/ipfs-uploader.ts +++ b/packages/storage/src/core/uploaders/ipfs-uploader.ts @@ -20,9 +20,11 @@ import FormData from "form-data"; export class IpfsUploader implements IStorageUploader { public gatewayUrls: GatewayUrls; + public uploadWithGatewayUrl: boolean; constructor(options?: IpfsUploaderOptions) { this.gatewayUrls = prepareGatewayUrls(options?.gatewayUrls); + this.uploadWithGatewayUrl = options?.uploadWithGatewayUrl || false; } async uploadBatch( diff --git a/packages/storage/src/types/upload.ts b/packages/storage/src/types/upload.ts index 22e987edda3..159fe0c43e6 100644 --- a/packages/storage/src/types/upload.ts +++ b/packages/storage/src/types/upload.ts @@ -5,6 +5,7 @@ export type UploadOptions = { [key: string]: any }; export interface IStorageUploader { gatewayUrls?: GatewayUrls; + uploadWithGatewayUrl?: boolean; uploadBatch(data: (string | FileOrBuffer)[], options?: T): Promise; } @@ -22,6 +23,7 @@ export type UploadProgressEvent = { export type IpfsUploaderOptions = { gatewayUrls?: GatewayUrls; + uploadWithGatewayUrl?: boolean; }; export type IpfsUploadBatchOptions = { diff --git a/packages/storage/test/ipfs.test.ts b/packages/storage/test/ipfs.test.ts index 66705e37ffc..854ac66f792 100644 --- a/packages/storage/test/ipfs.test.ts +++ b/packages/storage/test/ipfs.test.ts @@ -1,5 +1,5 @@ /* eslint-disable no-unused-expressions */ -import { ThirdwebStorage } from "../src"; +import { IpfsUploader, ThirdwebStorage } from "../src"; import { DEFAULT_GATEWAY_URLS } from "../src/common/urls"; import { expect } from "chai"; import { readFileSync } from "fs"; @@ -149,7 +149,7 @@ describe("IPFS", async () => { ); }); - it("Should upload without directory if specified", async () => { + it("Should upload without directory if specified on function", async () => { const uri = await storage.upload( { name: "Upload Without Directory", @@ -170,6 +170,31 @@ describe("IPFS", async () => { expect(json.description).to.equal("Uploading alone without a directory..."); }); + it("Should upload without directory if specified on class", async () => { + const solanaStorage = new ThirdwebStorage( + new IpfsUploader({ uploadWithGatewayUrl: true }), + ); + + const uri = await solanaStorage.upload( + { + name: "Upload Without Directory", + description: "Uploading alone without a directory...", + }, + { + uploadWithoutDirectory: true, + }, + ); + + expect(uri).to.equal( + "ipfs://QmdnBEP9UFcRfbuAyXFefNccNbuKWTscHrpWZatvqz9VcV", + ); + + const json = await storage.downloadJSON(uri); + + expect(json.name).to.equal("Upload Without Directory"); + expect(json.description).to.equal("Uploading alone without a directory..."); + }); + it("Should throw an error on upload without directory with multiple uploads", async () => { try { await storage.uploadBatch( From 5fbe6b9f91c3b8976efdb20735901847636c33ad Mon Sep 17 00:00:00 2001 From: adam-maj Date: Fri, 16 Sep 2022 15:03:56 -0700 Subject: [PATCH 10/19] Update storage usage in Solana --- .changeset/shy-toys-perform.md | 5 +++++ packages/solana/src/classes/deployer.ts | 13 ++++++------- packages/solana/src/contracts/nft-collection.ts | 8 ++++---- packages/solana/src/contracts/nft-drop.ts | 16 ++++++++++------ packages/solana/src/contracts/token.ts | 10 +++++++--- packages/solana/src/sdk.ts | 15 +++++---------- packages/solana/src/server/index.ts | 4 ++-- packages/solana/src/types/contracts/index.ts | 4 ++-- packages/solana/src/types/nft.ts | 8 ++++---- .../storage/src/core/uploaders/ipfs-uploader.ts | 6 +++--- packages/storage/src/types/data.ts | 7 +++++++ packages/storage/src/types/upload.ts | 4 ++-- 12 files changed, 57 insertions(+), 43 deletions(-) create mode 100644 .changeset/shy-toys-perform.md diff --git a/.changeset/shy-toys-perform.md b/.changeset/shy-toys-perform.md new file mode 100644 index 00000000000..916ad847974 --- /dev/null +++ b/.changeset/shy-toys-perform.md @@ -0,0 +1,5 @@ +--- +"@thirdweb-dev/storage": major +--- + +Upgrade to initial storage SDK with IPFS support diff --git a/packages/solana/src/classes/deployer.ts b/packages/solana/src/classes/deployer.ts index 9f175bc7d8a..fa5db6ae8d3 100644 --- a/packages/solana/src/classes/deployer.ts +++ b/packages/solana/src/classes/deployer.ts @@ -21,21 +21,20 @@ import { DataV2, } from "@metaplex-foundation/mpl-token-metadata"; import { Keypair } from "@solana/web3.js"; -import { IStorage } from "@thirdweb-dev/storage"; -import BN from "bn.js"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; export class Deployer { private metaplex: Metaplex; - private storage: IStorage; + private storage: ThirdwebStorage; - constructor(metaplex: Metaplex, storage: IStorage) { + constructor(metaplex: Metaplex, storage: ThirdwebStorage) { this.metaplex = metaplex; this.storage = storage; } async createToken(tokenMetadata: TokenMetadataInput): Promise { const tokenMetadataParsed = TokenMetadataInputSchema.parse(tokenMetadata); - const uri = await this.storage.uploadMetadata(tokenMetadataParsed); + const uri = await this.storage.upload(tokenMetadataParsed); const mint = Keypair.generate(); const owner = this.metaplex.identity().publicKey; const mintTx = await this.metaplex @@ -81,7 +80,7 @@ export class Deployer { collectionMetadata: NFTCollectionMetadataInput, ): Promise { const parsed = NFTCollectionMetadataInputSchema.parse(collectionMetadata); - const uri = await this.storage.uploadMetadata(parsed); + const uri = await this.storage.upload(parsed); const { nft: collectionNft } = await this.metaplex .nfts() @@ -103,7 +102,7 @@ export class Deployer { async createNftDrop(metadata: NFTDropMetadataInput): Promise { const collectionInfo = NFTCollectionMetadataInputSchema.parse(metadata); const candyMachineInfo = NFTDropConditionsOutputSchema.parse(metadata); - const uri = await this.storage.uploadMetadata(collectionInfo); + const uri = await this.storage.upload(collectionInfo); const collectionMint = Keypair.generate(); const collectionTx = await this.metaplex diff --git a/packages/solana/src/contracts/nft-collection.ts b/packages/solana/src/contracts/nft-collection.ts index b91e1cd3ec6..4bc6fdfdc71 100644 --- a/packages/solana/src/contracts/nft-collection.ts +++ b/packages/solana/src/contracts/nft-collection.ts @@ -16,11 +16,11 @@ import { Metadata, } from "@metaplex-foundation/mpl-token-metadata"; import { ConfirmedSignatureInfo, PublicKey } from "@solana/web3.js"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; export class NFTCollection { private metaplex: Metaplex; - private storage: IStorage; + private storage: ThirdwebStorage; private nft: NFTHelper; public publicKey: PublicKey; public accountType = "nft-collection" as const; @@ -28,7 +28,7 @@ export class NFTCollection { constructor( collectionMintAddress: string, metaplex: Metaplex, - storage: IStorage, + storage: ThirdwebStorage, ) { this.storage = storage; this.metaplex = metaplex; @@ -206,7 +206,7 @@ export class NFTCollection { // TODO add options param for initial/maximum supply async mintTo(to: string, metadata: NFTMetadataInput) { - const uri = await this.storage.uploadMetadata(metadata); + const uri = await this.storage.upload(metadata); const { nft } = await this.metaplex .nfts() .create({ diff --git a/packages/solana/src/contracts/nft-drop.ts b/packages/solana/src/contracts/nft-drop.ts index a9a1114e5d4..49f8b11e201 100644 --- a/packages/solana/src/contracts/nft-drop.ts +++ b/packages/solana/src/contracts/nft-drop.ts @@ -9,18 +9,22 @@ import { } from "../types/nft"; import { Metaplex } from "@metaplex-foundation/js"; import { PublicKey } from "@solana/web3.js"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import invariant from "tiny-invariant"; export class NFTDrop { private metaplex: Metaplex; - private storage: IStorage; + private storage: ThirdwebStorage; private nft: NFTHelper; public accountType = "nft-drop" as const; public publicKey: PublicKey; public claimConditions: ClaimConditions; - constructor(dropMintAddress: string, metaplex: Metaplex, storage: IStorage) { + constructor( + dropMintAddress: string, + metaplex: Metaplex, + storage: ThirdwebStorage, + ) { this.storage = storage; this.metaplex = metaplex; this.nft = new NFTHelper(metaplex); @@ -47,7 +51,7 @@ export class NFTDrop { const info = await this.getCandyMachine(); const nfts = await Promise.all( info.items.map(async (item) => { - const metadata = await this.storage.get(item.uri); + const metadata = await this.storage.downloadJSON(item.uri); return { uri: item.uri, ...metadata }; }), ); @@ -97,8 +101,8 @@ export class NFTDrop { const parsedMetadatas = metadatas.map((metadata) => CommonNFTInput.parse(metadata), ); - const upload = await this.storage.uploadMetadataBatch(parsedMetadatas); - const items = upload.uris.map((uri, i) => ({ + const uris = await this.storage.uploadBatch(parsedMetadatas); + const items = uris.map((uri, i) => ({ name: parsedMetadatas[i].name || "", uri, })); diff --git a/packages/solana/src/contracts/token.ts b/packages/solana/src/contracts/token.ts index 98368044ca7..5b5cd4bb23f 100644 --- a/packages/solana/src/contracts/token.ts +++ b/packages/solana/src/contracts/token.ts @@ -16,16 +16,20 @@ import { } from "@metaplex-foundation/js"; import { getAccount, getAssociatedTokenAddress } from "@solana/spl-token"; import { Connection, PublicKey } from "@solana/web3.js"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; export class Token { private connection: Connection; private metaplex: Metaplex; - private storage: IStorage; + private storage: ThirdwebStorage; public accountType = "token" as const; public publicKey: PublicKey; - constructor(tokenMintAddress: string, metaplex: Metaplex, storage: IStorage) { + constructor( + tokenMintAddress: string, + metaplex: Metaplex, + storage: ThirdwebStorage, + ) { this.storage = storage; this.metaplex = metaplex; this.connection = metaplex.connection; diff --git a/packages/solana/src/sdk.ts b/packages/solana/src/sdk.ts index ec4154cc998..7ff08923de4 100644 --- a/packages/solana/src/sdk.ts +++ b/packages/solana/src/sdk.ts @@ -1,7 +1,6 @@ import { Deployer } from "./classes/deployer"; import { Registry } from "./classes/registry"; import { UserWallet } from "./classes/user-wallet"; -import { DEFAULT_IPFS_GATEWAY } from "./constants/urls"; import { NFTCollection } from "./contracts/nft-collection"; import { NFTDrop } from "./contracts/nft-drop"; import { Program } from "./contracts/program"; @@ -16,17 +15,17 @@ import { setProvider, } from "@project-serum/anchor"; import { Connection } from "@solana/web3.js"; -import { IpfsStorage, IStorage, PinataUploader } from "@thirdweb-dev/storage"; +import { IpfsUploader, ThirdwebStorage } from "@thirdweb-dev/storage"; export class ThirdwebSDK { - static fromNetwork(network: Network, storage?: IStorage): ThirdwebSDK { + static fromNetwork(network: Network, storage?: ThirdwebStorage): ThirdwebSDK { return new ThirdwebSDK(new Connection(getUrlForNetwork(network)), storage); } private connection: Connection; private metaplex: Metaplex; private anchorProvider: AnchorProvider; - private storage: IStorage; + private storage: ThirdwebStorage; public registry: Registry; public deployer: Deployer; @@ -34,12 +33,8 @@ export class ThirdwebSDK { constructor( connection: Connection, - storage: IStorage = new IpfsStorage( - DEFAULT_IPFS_GATEWAY, - new PinataUploader(), - { - appendGatewayUrl: true, - }, + storage: ThirdwebStorage = new ThirdwebStorage( + new IpfsUploader({ uploadWithGatewayUrl: true }), ), ) { this.connection = connection; diff --git a/packages/solana/src/server/index.ts b/packages/solana/src/server/index.ts index 95168472b9b..595ec78f208 100644 --- a/packages/solana/src/server/index.ts +++ b/packages/solana/src/server/index.ts @@ -2,11 +2,11 @@ import { ThirdwebSDK } from "../sdk"; import { Network } from "../types/index"; import { getPayer } from "./local-config"; import { KeypairSigner } from "@metaplex-foundation/js"; -import { IpfsStorage, IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; export function createThirdwebSDK( network: Network, - storage: IStorage = new IpfsStorage(), + storage: ThirdwebStorage = new ThirdwebStorage(), ): ThirdwebSDK { const payer = getPayer(); const signer: KeypairSigner = { diff --git a/packages/solana/src/types/contracts/index.ts b/packages/solana/src/types/contracts/index.ts index 3d47de70ded..67a40ce4494 100644 --- a/packages/solana/src/types/contracts/index.ts +++ b/packages/solana/src/types/contracts/index.ts @@ -1,5 +1,5 @@ import { AmountSchema, JsonSchema } from "../common"; -import { FileBufferOrStringSchema } from "@thirdweb-dev/storage"; +import { FileOrBufferOrStringSchema } from "@thirdweb-dev/storage"; import { z } from "zod"; /** @@ -9,7 +9,7 @@ export const CommonContractSchema = z.object({ name: z.string(), symbol: z.string().optional(), description: z.string().optional(), - image: FileBufferOrStringSchema.optional(), + image: FileOrBufferOrStringSchema.optional(), external_link: z.string().url().optional(), }); diff --git a/packages/solana/src/types/nft.ts b/packages/solana/src/types/nft.ts index d453b8c934c..a21bc519ea5 100644 --- a/packages/solana/src/types/nft.ts +++ b/packages/solana/src/types/nft.ts @@ -1,5 +1,5 @@ import { HexColor, JsonSchema, OptionalPropertiesInput } from "./common"; -import { FileBufferOrStringSchema } from "@thirdweb-dev/storage"; +import { FileOrBufferOrStringSchema } from "@thirdweb-dev/storage"; import { z } from "zod"; /** @@ -10,8 +10,8 @@ export const CommonTokenInput = z name: z.string().optional(), symbol: z.string().optional(), description: z.string().nullable().optional(), - image: FileBufferOrStringSchema.nullable().optional(), - external_url: FileBufferOrStringSchema.nullable().optional(), + image: FileOrBufferOrStringSchema.nullable().optional(), + external_url: FileOrBufferOrStringSchema.nullable().optional(), }) .catchall(z.lazy(() => JsonSchema)); @@ -29,7 +29,7 @@ export const CommonTokenOutput = CommonTokenInput.extend({ * @internal */ export const CommonNFTInput = CommonTokenInput.extend({ - animation_url: FileBufferOrStringSchema.optional(), + animation_url: FileOrBufferOrStringSchema.optional(), background_color: HexColor.optional(), properties: OptionalPropertiesInput, }); diff --git a/packages/storage/src/core/uploaders/ipfs-uploader.ts b/packages/storage/src/core/uploaders/ipfs-uploader.ts index a0e1be05fee..67d12c1151d 100644 --- a/packages/storage/src/core/uploaders/ipfs-uploader.ts +++ b/packages/storage/src/core/uploaders/ipfs-uploader.ts @@ -9,7 +9,7 @@ import { isFileInstance, } from "../../common/utils"; import { - FileOrBuffer, + FileOrBufferOrString, GatewayUrls, IpfsUploadBatchOptions, IpfsUploaderOptions, @@ -28,7 +28,7 @@ export class IpfsUploader implements IStorageUploader { } async uploadBatch( - data: (string | FileOrBuffer)[], + data: FileOrBufferOrString[], options?: IpfsUploadBatchOptions, ): Promise { if (options?.uploadWithoutDirectory && data.length > 1) { @@ -69,7 +69,7 @@ export class IpfsUploader implements IStorageUploader { private buildFormData( form: FormData, - files: (string | FileOrBuffer)[], + files: FileOrBufferOrString[], options?: IpfsUploadBatchOptions, ) { const fileNames: string[] = []; diff --git a/packages/storage/src/types/data.ts b/packages/storage/src/types/data.ts index 769fde1cd48..175acffde89 100644 --- a/packages/storage/src/types/data.ts +++ b/packages/storage/src/types/data.ts @@ -32,6 +32,13 @@ export type BufferOrStringWithName = { name: string; }; +export const FileOrBufferOrStringSchema = z.union([ + FileOrBufferSchema, + z.string(), +]); + +export type FileOrBufferOrString = FileOrBuffer | string; + const JsonSchema: z.ZodType = z.lazy(() => z.union([ JsonLiteralSchema, diff --git a/packages/storage/src/types/upload.ts b/packages/storage/src/types/upload.ts index 159fe0c43e6..5d702a429ea 100644 --- a/packages/storage/src/types/upload.ts +++ b/packages/storage/src/types/upload.ts @@ -1,4 +1,4 @@ -import { FileOrBuffer } from "./data"; +import { FileOrBufferOrString } from "./data"; import { GatewayUrls } from "./download"; export type UploadOptions = { [key: string]: any }; @@ -6,7 +6,7 @@ export type UploadOptions = { [key: string]: any }; export interface IStorageUploader { gatewayUrls?: GatewayUrls; uploadWithGatewayUrl?: boolean; - uploadBatch(data: (string | FileOrBuffer)[], options?: T): Promise; + uploadBatch(data: FileOrBufferOrString[], options?: T): Promise; } export type UploadProgressEvent = { From 5ef68402b163da0ba9394342f204fa804325f7b9 Mon Sep 17 00:00:00 2001 From: adam-maj Date: Fri, 16 Sep 2022 16:00:02 -0700 Subject: [PATCH 11/19] Update storage usage in SDK --- packages/sdk/src/common/claim-conditions.ts | 14 +++--- packages/sdk/src/common/feature-detection.ts | 30 ++++++------ packages/sdk/src/common/nft.ts | 49 +++++++++++++------ packages/sdk/src/common/snapshots.ts | 6 +-- packages/sdk/src/contracts/index.ts | 4 +- .../prebuilt-implementations/edition-drop.ts | 4 +- .../prebuilt-implementations/edition.ts | 4 +- .../prebuilt-implementations/marketplace.ts | 6 +-- .../prebuilt-implementations/multiwrap.ts | 4 +- .../nft-collection.ts | 4 +- .../prebuilt-implementations/nft-drop.ts | 6 +-- .../prebuilt-implementations/pack.ts | 4 +- .../signature-drop.ts | 4 +- .../prebuilt-implementations/split.ts | 6 +-- .../prebuilt-implementations/token-drop.ts | 4 +- .../prebuilt-implementations/token.ts | 4 +- .../prebuilt-implementations/vote.ts | 6 +-- packages/sdk/src/contracts/smart-contract.ts | 4 +- .../sdk/src/core/classes/contract-deployer.ts | 6 +-- .../sdk/src/core/classes/contract-metadata.ts | 8 +-- .../classes/contract-published-metadata.ts | 9 ++-- .../src/core/classes/contract-publisher.ts | 20 ++++---- .../sdk/src/core/classes/delayed-reveal.ts | 40 ++++++++------- .../src/core/classes/drop-claim-conditions.ts | 6 +-- .../classes/drop-erc1155-claim-conditions.ts | 6 +-- .../core/classes/erc-1155-batch-mintable.ts | 6 +-- .../erc-1155-claimable-with-conditions.ts | 6 +-- .../src/core/classes/erc-1155-lazymintable.ts | 8 ++- .../sdk/src/core/classes/erc-1155-mintable.ts | 6 +-- .../classes/erc-1155-signature-mintable.ts | 6 +-- .../sdk/src/core/classes/erc-1155-standard.ts | 6 +-- packages/sdk/src/core/classes/erc-1155.ts | 6 +-- .../sdk/src/core/classes/erc-20-claimable.ts | 6 +-- .../sdk/src/core/classes/erc-20-droppable.ts | 6 +-- .../sdk/src/core/classes/erc-20-standard.ts | 7 ++- packages/sdk/src/core/classes/erc-20.ts | 6 +-- .../core/classes/erc-721-batch-mintable.ts | 6 +-- .../erc-721-claimable-with-conditions.ts | 6 +-- .../src/core/classes/erc-721-lazymintable.ts | 21 +++----- .../sdk/src/core/classes/erc-721-mintable.ts | 6 +-- .../sdk/src/core/classes/erc-721-standard.ts | 6 +-- ...rc-721-with-quantity-signature-mintable.ts | 6 +-- packages/sdk/src/core/classes/erc-721.ts | 6 +-- packages/sdk/src/core/classes/factory.ts | 12 ++--- .../src/core/classes/marketplace-auction.ts | 6 +-- .../src/core/classes/marketplace-direct.ts | 6 +-- packages/sdk/src/core/sdk.ts | 14 +++--- .../sdk/src/schema/contracts/common/index.ts | 4 +- packages/sdk/src/schema/contracts/custom.ts | 8 +-- .../sdk/src/schema/tokens/common/index.ts | 8 +-- .../sdk/src/types/deploy/deploy-metadata.ts | 14 +++--- packages/sdk/test/edition.test.ts | 4 +- packages/sdk/test/hooks.ts | 4 +- packages/sdk/test/mock/MockStorage.ts | 14 +++--- packages/sdk/test/nft-drop.test.ts | 2 +- packages/sdk/test/nft.test.ts | 4 +- packages/sdk/test/publisher.test.ts | 14 ++++-- packages/sdk/test/signature-drop.test.ts | 6 +-- packages/sdk/test/signature-mint-1155.test.ts | 6 +-- packages/sdk/test/signature-mint-721.test.ts | 6 +-- packages/sdk/test/snapshot.test.ts | 4 +- packages/storage/src/core/storage.ts | 25 +++++----- packages/storage/src/types/data.ts | 2 +- 63 files changed, 279 insertions(+), 268 deletions(-) diff --git a/packages/sdk/src/common/claim-conditions.ts b/packages/sdk/src/common/claim-conditions.ts index 0579f7bd203..58edcf22449 100644 --- a/packages/sdk/src/common/claim-conditions.ts +++ b/packages/sdk/src/common/claim-conditions.ts @@ -25,7 +25,7 @@ import { } from "./currency"; import { createSnapshot } from "./snapshots"; import { IDropClaimCondition } from "@thirdweb-dev/contracts-js/dist/declarations/src/DropERC20"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, @@ -47,7 +47,7 @@ export async function prepareClaim( merkleMetadataFetcher: () => Promise>, tokenDecimals: number, contractWrapper: ContractWrapper, - storage: IStorage, + storage: ThirdwebStorage, checkERC20Allowance: boolean, ): Promise { const addressToClaim = await contractWrapper.getSignerAddress(); @@ -131,7 +131,7 @@ type Snapshot = export async function fetchSnapshot( merkleRoot: string, merkleMetadata: Record | undefined, - storage: IStorage, + storage: ThirdwebStorage, ): Promise { if (!merkleMetadata) { return undefined; @@ -139,7 +139,7 @@ export async function fetchSnapshot( const snapshotUri = merkleMetadata[merkleRoot]; let snapshot: Snapshot = undefined; if (snapshotUri) { - const raw = await storage.get(snapshotUri); + const raw = await storage.downloadJSON(snapshotUri); const snapshotData = SnapshotSchema.parse(raw); if (merkleRoot === snapshotData.merkleRoot) { snapshot = snapshotData.claims; @@ -211,7 +211,7 @@ export async function getClaimerProofs( merkleRoot: string, tokenDecimals: number, merkleMetadata: Record, - storage: IStorage, + storage: ThirdwebStorage, ): Promise<{ maxClaimable: BigNumber; proof: string[] }> { const claims: Snapshot = await fetchSnapshot( merkleRoot, @@ -252,7 +252,7 @@ export async function processClaimConditionInputs( claimConditionInputs: ClaimConditionInput[], tokenDecimals: number, provider: providers.Provider, - storage: IStorage, + storage: ThirdwebStorage, ) { const snapshotInfos: SnapshotInfo[] = []; const inputsWithSnapshots = await Promise.all( @@ -355,7 +355,7 @@ export async function transformResultToClaimCondition( tokenDecimals: number, provider: providers.Provider, merkleMetadata: Record | undefined, - storage: IStorage, + storage: ThirdwebStorage, ): Promise { const cv = await fetchCurrencyValue(provider, pm.currency, pm.pricePerToken); diff --git a/packages/sdk/src/common/feature-detection.ts b/packages/sdk/src/common/feature-detection.ts index 2f7291262d9..49fcb8c4b2c 100644 --- a/packages/sdk/src/common/feature-detection.ts +++ b/packages/sdk/src/common/feature-detection.ts @@ -21,7 +21,7 @@ import { PublishedMetadata, } from "../schema/contracts/custom"; import { ExtensionNotImplementedError } from "./error"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BaseContract, ethers } from "ethers"; import { z } from "zod"; @@ -57,7 +57,7 @@ function matchesAbiInterface( */ export async function extractConstructorParams( predeployMetadataUri: string, - storage: IStorage, + storage: ThirdwebStorage, ) { const meta = await fetchPreDeployMetadata(predeployMetadataUri, storage); return extractConstructorParamsFromAbi(meta.abi); @@ -70,7 +70,7 @@ export async function extractConstructorParams( */ export async function extractFunctions( predeployMetadataUri: string, - storage: IStorage, + storage: ThirdwebStorage, ): Promise { const metadata = await fetchPreDeployMetadata(predeployMetadataUri, storage); return extractFunctionsFromAbi(metadata.abi, metadata.metadata); @@ -328,7 +328,7 @@ function isHexStrict(hex: string | number) { export async function fetchContractMetadataFromAddress( address: string, provider: ethers.providers.Provider, - storage: IStorage, + storage: ThirdwebStorage, ) { const compilerMetadataUri = await resolveContractUriFromAddress( address, @@ -347,9 +347,9 @@ export async function fetchContractMetadataFromAddress( */ export async function fetchContractMetadata( compilerMetadataUri: string, - storage: IStorage, + storage: ThirdwebStorage, ): Promise { - const metadata = await storage.get(compilerMetadataUri); + const metadata = await storage.downloadJSON(compilerMetadataUri); const abi = AbiSchema.parse(metadata.output.abi); const compilationTarget = metadata.settings.compilationTarget; const targets = Object.keys(compilationTarget); @@ -381,7 +381,7 @@ export async function fetchContractMetadata( */ export async function fetchSourceFilesFromMetadata( publishedMetadata: PublishedMetadata, - storage: IStorage, + storage: ThirdwebStorage, ): Promise { return await Promise.all( Object.entries(publishedMetadata.metadata.sources).map( @@ -395,7 +395,7 @@ export async function fetchSourceFilesFromMetadata( setTimeout(() => rej("timeout"), 5000), ); const source = await Promise.race([ - storage.getRaw(`ipfs://${ipfsHash}`), + (await storage.download(`ipfs://${ipfsHash}`)).text(), timeout, ]); return { @@ -420,10 +420,10 @@ export async function fetchSourceFilesFromMetadata( */ export async function fetchRawPredeployMetadata( publishMetadataUri: string, - storage: IStorage, + storage: ThirdwebStorage, ) { return PreDeployMetadata.parse( - JSON.parse(await storage.getRaw(publishMetadataUri)), + JSON.parse(await (await storage.download(publishMetadataUri)).text()), ); } @@ -435,10 +435,12 @@ export async function fetchRawPredeployMetadata( */ export async function fetchPreDeployMetadata( publishMetadataUri: string, - storage: IStorage, + storage: ThirdwebStorage, ): Promise { const rawMeta = await fetchRawPredeployMetadata(publishMetadataUri, storage); - const deployBytecode = await storage.getRaw(rawMeta.bytecodeUri); + const deployBytecode = await ( + await storage.download(rawMeta.bytecodeUri) + ).text(); const parsedMeta = await fetchContractMetadata(rawMeta.metadataUri, storage); return PreDeployMetadataFetchedSchema.parse({ ...rawMeta, @@ -455,9 +457,9 @@ export async function fetchPreDeployMetadata( */ export async function fetchExtendedReleaseMetadata( publishMetadataUri: string, - storage: IStorage, + storage: ThirdwebStorage, ): Promise { - const meta = await storage.getRaw(publishMetadataUri); + const meta = await (await storage.download(publishMetadataUri)).text(); return FullPublishMetadataSchemaOutput.parse(JSON.parse(meta)); } diff --git a/packages/sdk/src/common/nft.ts b/packages/sdk/src/common/nft.ts index 7fc861cc178..373494dcd54 100644 --- a/packages/sdk/src/common/nft.ts +++ b/packages/sdk/src/common/nft.ts @@ -9,7 +9,6 @@ import { NFTMetadataInput, NFTMetadataOrUri, } from "../schema/tokens/common"; -import { UploadProgressEvent } from "../types/index"; import { NotFoundError } from "./error"; import type { IERC1155Metadata, @@ -19,7 +18,10 @@ import type { import ERC165MetadataAbi from "@thirdweb-dev/contracts-js/dist/abis/IERC165.json"; import ERC721MetadataAbi from "@thirdweb-dev/contracts-js/dist/abis/IERC721Metadata.json"; import ERC1155MetadataAbi from "@thirdweb-dev/contracts-js/dist/abis/IERC1155Metadata.json"; -import type { IStorage } from "@thirdweb-dev/storage"; +import type { + ThirdwebStorage, + UploadProgressEvent, +} from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, Contract, ethers, providers } from "ethers"; const FALLBACK_METADATA = { @@ -37,7 +39,7 @@ const FALLBACK_METADATA = { export async function fetchTokenMetadata( tokenId: BigNumberish, tokenUri: string, - storage: IStorage, + storage: ThirdwebStorage, ): Promise { const parsedUri = tokenUri.replace( "{id}", @@ -45,14 +47,14 @@ export async function fetchTokenMetadata( ); let jsonMetadata; try { - jsonMetadata = await storage.get(parsedUri); + jsonMetadata = await storage.downloadJSON(parsedUri); } catch (err) { const unparsedTokenIdUri = tokenUri.replace( "{id}", BigNumber.from(tokenId).toString(), ); try { - jsonMetadata = await storage.get(unparsedTokenIdUri); + jsonMetadata = await storage.downloadJSON(unparsedTokenIdUri); } catch (e) { console.warn( `failed to get token metadata: ${JSON.stringify({ @@ -83,7 +85,7 @@ export async function fetchTokenMetadataForContract( contractAddress: string, provider: providers.Provider, tokenId: BigNumberish, - storage: IStorage, + storage: ThirdwebStorage, ) { let uri: string | undefined; const erc165 = new Contract( @@ -123,12 +125,12 @@ export async function fetchTokenMetadataForContract( */ export async function uploadOrExtractURI( metadata: NFTMetadataOrUri, - storage: IStorage, + storage: ThirdwebStorage, ): Promise { if (typeof metadata === "string") { return metadata; } else { - return await storage.uploadMetadata(CommonNFTInput.parse(metadata)); + return await storage.upload(CommonNFTInput.parse(metadata)); } } @@ -143,10 +145,8 @@ export async function uploadOrExtractURI( */ export async function uploadOrExtractURIs( metadatas: NFTMetadataOrUri[], - storage: IStorage, + storage: ThirdwebStorage, startNumber?: number, - contractAddress?: string, - signerAddress?: string, options?: { onProgress: (event: UploadProgressEvent) => void; }, @@ -154,12 +154,14 @@ export async function uploadOrExtractURIs( if (isUriList(metadatas)) { return metadatas; } else if (isMetadataList(metadatas)) { - const { uris } = await storage.uploadMetadataBatch( + const uris = await storage.uploadBatch( metadatas.map((m) => CommonNFTInput.parse(m)), - startNumber, - contractAddress, - signerAddress, - options, + { + rewriteFileNames: { + fileStartNumber: startNumber || 0, + }, + onProgress: options?.onProgress, + }, ); return uris; } else { @@ -169,6 +171,21 @@ export async function uploadOrExtractURIs( } } +export function getBaseUriFromBatch(uris: string[]): string { + const baseUri = uris[0].substring(0, uris[0].lastIndexOf("/")); + for (let i = 0; i < uris.length; i++) { + const uri = uris[i].substring(0, uris[i].lastIndexOf("/")); + if (baseUri !== uri) { + throw new Error( + `Can only create batches with the same base URI for every entry in the batch. Expected '${baseUri}' but got '${uri}'`, + ); + } + } + + // Ensure that baseUri ends with trailing slash + return baseUri.replace(/\/$/, "") + "/"; +} + function isUriList(metadatas: NFTMetadataOrUri[]): metadatas is string[] { return metadatas.find((m) => typeof m !== "string") === undefined; } diff --git a/packages/sdk/src/common/snapshots.ts b/packages/sdk/src/common/snapshots.ts index d57b09b038f..202912ef5d1 100644 --- a/packages/sdk/src/common/snapshots.ts +++ b/packages/sdk/src/common/snapshots.ts @@ -7,7 +7,7 @@ import { SnapshotInput, } from "../types/claim-conditions/claim-conditions"; import { DuplicateLeafsError } from "./error"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, utils } from "ethers"; /** @@ -21,7 +21,7 @@ import { BigNumber, BigNumberish, utils } from "ethers"; export async function createSnapshot( snapshotInput: SnapshotInput, tokenDecimals: number, - storage: IStorage, + storage: ThirdwebStorage, ): Promise { const input = SnapshotInputSchema.parse(snapshotInput); const addresses = input.map((i) => i.address); @@ -51,7 +51,7 @@ export async function createSnapshot( }), }); - const uri = await storage.uploadMetadata(snapshot); + const uri = await storage.upload(snapshot); return { merkleRoot: tree.getHexRoot(), snapshotUri: uri, diff --git a/packages/sdk/src/contracts/index.ts b/packages/sdk/src/contracts/index.ts index 0c2f55d90ac..63669930e4a 100644 --- a/packages/sdk/src/contracts/index.ts +++ b/packages/sdk/src/contracts/index.ts @@ -15,12 +15,12 @@ import { import { CustomContractSchema } from "../schema/contracts/custom"; import { DropErc20ContractSchema } from "../schema/contracts/drop-erc20"; import { MultiwrapContractSchema } from "../schema/contracts/multiwrap"; -import type { IStorage } from "@thirdweb-dev/storage"; +import type { ThirdwebStorage } from "@thirdweb-dev/storage"; type InitalizeParams = [ network: NetworkOrSignerOrProvider, address: string, - storage: IStorage, + storage: ThirdwebStorage, options?: SDKOptions, ]; diff --git a/packages/sdk/src/contracts/prebuilt-implementations/edition-drop.ts b/packages/sdk/src/contracts/prebuilt-implementations/edition-drop.ts index 5e6599ade01..8f8c1fbe516 100644 --- a/packages/sdk/src/contracts/prebuilt-implementations/edition-drop.ts +++ b/packages/sdk/src/contracts/prebuilt-implementations/edition-drop.ts @@ -26,7 +26,7 @@ import { NFTMetadata, NFTMetadataOrUri } from "../../schema/tokens/common"; import { QueryAllParams, UploadProgressEvent } from "../../types"; import type { DropERC1155 } from "@thirdweb-dev/contracts-js"; import type ABI from "@thirdweb-dev/contracts-js/dist/abis/DropERC1155.json"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, CallOverrides, constants } from "ethers"; /** @@ -113,7 +113,7 @@ export class EditionDropImpl extends StandardErc1155 { constructor( network: NetworkOrSignerOrProvider, address: string, - storage: IStorage, + storage: ThirdwebStorage, options: SDKOptions = {}, abi: typeof ABI, contractWrapper = new ContractWrapper( diff --git a/packages/sdk/src/contracts/prebuilt-implementations/edition.ts b/packages/sdk/src/contracts/prebuilt-implementations/edition.ts index 205984bdba0..ed4b698a2e9 100644 --- a/packages/sdk/src/contracts/prebuilt-implementations/edition.ts +++ b/packages/sdk/src/contracts/prebuilt-implementations/edition.ts @@ -27,7 +27,7 @@ import { import { QueryAllParams } from "../../types"; import type { TokenERC1155 } from "@thirdweb-dev/contracts-js"; import type ABI from "@thirdweb-dev/contracts-js/dist/abis/TokenERC1155.json"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, CallOverrides, constants } from "ethers"; /** @@ -103,7 +103,7 @@ export class EditionImpl extends StandardErc1155 { constructor( network: NetworkOrSignerOrProvider, address: string, - storage: IStorage, + storage: ThirdwebStorage, options: SDKOptions = {}, abi: typeof ABI, contractWrapper = new ContractWrapper( diff --git a/packages/sdk/src/contracts/prebuilt-implementations/marketplace.ts b/packages/sdk/src/contracts/prebuilt-implementations/marketplace.ts index b4d115b9751..4afce5d6526 100644 --- a/packages/sdk/src/contracts/prebuilt-implementations/marketplace.ts +++ b/packages/sdk/src/contracts/prebuilt-implementations/marketplace.ts @@ -20,7 +20,7 @@ import { AuctionListing, DirectListing } from "../../types/marketplace"; import { MarketplaceFilter } from "../../types/marketplace/MarketPlaceFilter"; import type { Marketplace as MarketplaceContract } from "@thirdweb-dev/contracts-js"; import type ABI from "@thirdweb-dev/contracts-js/dist/abis/Marketplace.json"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, CallOverrides, constants } from "ethers"; import invariant from "tiny-invariant"; @@ -43,7 +43,7 @@ export class MarketplaceImpl implements UpdateableNetwork { public abi: typeof ABI; private contractWrapper: ContractWrapper; - private storage: IStorage; + private storage: ThirdwebStorage; public encoder: ContractEncoder; public events: ContractEvents; @@ -135,7 +135,7 @@ export class MarketplaceImpl implements UpdateableNetwork { constructor( network: NetworkOrSignerOrProvider, address: string, - storage: IStorage, + storage: ThirdwebStorage, options: SDKOptions = {}, abi: typeof ABI, contractWrapper = new ContractWrapper( diff --git a/packages/sdk/src/contracts/prebuilt-implementations/multiwrap.ts b/packages/sdk/src/contracts/prebuilt-implementations/multiwrap.ts index 5b38f416b3e..59e6ef4f17d 100644 --- a/packages/sdk/src/contracts/prebuilt-implementations/multiwrap.ts +++ b/packages/sdk/src/contracts/prebuilt-implementations/multiwrap.ts @@ -33,7 +33,7 @@ import { ITokenBundle, TokensWrappedEvent, } from "@thirdweb-dev/contracts-js/dist/declarations/src/Multiwrap"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumberish, CallOverrides, ethers } from "ethers"; /** @@ -91,7 +91,7 @@ export class MultiwrapImpl extends StandardErc721 { constructor( network: NetworkOrSignerOrProvider, address: string, - storage: IStorage, + storage: ThirdwebStorage, options: SDKOptions = {}, abi: typeof ABI, contractWrapper = new ContractWrapper( diff --git a/packages/sdk/src/contracts/prebuilt-implementations/nft-collection.ts b/packages/sdk/src/contracts/prebuilt-implementations/nft-collection.ts index 96dff7e05d5..09bd3c5ebf7 100644 --- a/packages/sdk/src/contracts/prebuilt-implementations/nft-collection.ts +++ b/packages/sdk/src/contracts/prebuilt-implementations/nft-collection.ts @@ -22,7 +22,7 @@ import { TokenErc721ContractSchema } from "../../schema/contracts/token-erc721"; import { SDKOptions } from "../../schema/sdk-options"; import type { TokenERC721 } from "@thirdweb-dev/contracts-js"; import type ABI from "@thirdweb-dev/contracts-js/dist/abis/TokenERC721.json"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumberish, CallOverrides, constants } from "ethers"; /** @@ -103,7 +103,7 @@ export class NFTCollectionImpl extends StandardErc721 { constructor( network: NetworkOrSignerOrProvider, address: string, - storage: IStorage, + storage: ThirdwebStorage, options: SDKOptions = {}, abi: typeof ABI, contractWrapper = new ContractWrapper( diff --git a/packages/sdk/src/contracts/prebuilt-implementations/nft-drop.ts b/packages/sdk/src/contracts/prebuilt-implementations/nft-drop.ts index 5f0dbcb314a..34edb9fe680 100644 --- a/packages/sdk/src/contracts/prebuilt-implementations/nft-drop.ts +++ b/packages/sdk/src/contracts/prebuilt-implementations/nft-drop.ts @@ -41,7 +41,7 @@ import { TokensClaimedEvent, TokensLazyMintedEvent, } from "@thirdweb-dev/contracts-js/dist/declarations/src/DropERC721"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, @@ -166,7 +166,7 @@ export class NFTDropImpl extends StandardErc721 { constructor( network: NetworkOrSignerOrProvider, address: string, - storage: IStorage, + storage: ThirdwebStorage, options: SDKOptions = {}, abi: typeof ABI, contractWrapper = new ContractWrapper( @@ -394,8 +394,6 @@ export class NFTDropImpl extends StandardErc721 { metadatas, this.storage, startFileNumber.toNumber(), - this.contractWrapper.readContract.address, - await this.contractWrapper.getSigner()?.getAddress(), options, ); // ensure baseUri is the same for the entire batch diff --git a/packages/sdk/src/contracts/prebuilt-implementations/pack.ts b/packages/sdk/src/contracts/prebuilt-implementations/pack.ts index 3506b7f49f1..a12d42ad162 100644 --- a/packages/sdk/src/contracts/prebuilt-implementations/pack.ts +++ b/packages/sdk/src/contracts/prebuilt-implementations/pack.ts @@ -39,7 +39,7 @@ import { PackCreatedEvent, PackOpenedEvent, } from "@thirdweb-dev/contracts-js/dist/declarations/src/Pack"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, CallOverrides, ethers } from "ethers"; /** @@ -96,7 +96,7 @@ export class PackImpl extends StandardErc1155 { constructor( network: NetworkOrSignerOrProvider, address: string, - storage: IStorage, + storage: ThirdwebStorage, options: SDKOptions = {}, abi: typeof ABI, contractWrapper = new ContractWrapper( diff --git a/packages/sdk/src/contracts/prebuilt-implementations/signature-drop.ts b/packages/sdk/src/contracts/prebuilt-implementations/signature-drop.ts index ae7a338393b..3bf4eadbded 100644 --- a/packages/sdk/src/contracts/prebuilt-implementations/signature-drop.ts +++ b/packages/sdk/src/contracts/prebuilt-implementations/signature-drop.ts @@ -35,7 +35,7 @@ import { } from "../../types/QueryParams"; import type { SignatureDrop as SignatureDropContract } from "@thirdweb-dev/contracts-js"; import type ABI from "@thirdweb-dev/contracts-js/dist/abis/SignatureDrop.json"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, CallOverrides, constants } from "ethers"; /** @@ -164,7 +164,7 @@ export class SignatureDropImpl extends StandardErc721 { constructor( network: NetworkOrSignerOrProvider, address: string, - storage: IStorage, + storage: ThirdwebStorage, options: SDKOptions = {}, abi: typeof ABI, contractWrapper = new ContractWrapper( diff --git a/packages/sdk/src/contracts/prebuilt-implementations/split.ts b/packages/sdk/src/contracts/prebuilt-implementations/split.ts index fde0c08e4fa..61f51bea73b 100644 --- a/packages/sdk/src/contracts/prebuilt-implementations/split.ts +++ b/packages/sdk/src/contracts/prebuilt-implementations/split.ts @@ -18,7 +18,7 @@ import type { } from "@thirdweb-dev/contracts-js"; import ERC20Abi from "@thirdweb-dev/contracts-js/dist/abis/IERC20.json"; import type ABI from "@thirdweb-dev/contracts-js/dist/abis/Split.json"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, CallOverrides, Contract } from "ethers"; /** @@ -39,7 +39,7 @@ export class SplitImpl implements UpdateableNetwork { static contractRoles = ["admin"] as const; private contractWrapper: ContractWrapper; - private storage: IStorage; + private storage: ThirdwebStorage; public abi: typeof ABI; public metadata: ContractMetadata; @@ -58,7 +58,7 @@ export class SplitImpl implements UpdateableNetwork { constructor( network: NetworkOrSignerOrProvider, address: string, - storage: IStorage, + storage: ThirdwebStorage, options: SDKOptions = {}, abi: typeof ABI, contractWrapper = new ContractWrapper( diff --git a/packages/sdk/src/contracts/prebuilt-implementations/token-drop.ts b/packages/sdk/src/contracts/prebuilt-implementations/token-drop.ts index 9a8f1168071..257bac95029 100644 --- a/packages/sdk/src/contracts/prebuilt-implementations/token-drop.ts +++ b/packages/sdk/src/contracts/prebuilt-implementations/token-drop.ts @@ -16,7 +16,7 @@ import { SDKOptions } from "../../schema/sdk-options"; import { Amount, CurrencyValue } from "../../types"; import type { DropERC20 } from "@thirdweb-dev/contracts-js"; import type ABI from "@thirdweb-dev/contracts-js/dist/abis/DropERC20.json"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { CallOverrides, constants } from "ethers"; /** @@ -77,7 +77,7 @@ export class TokenDropImpl extends StandardErc20 { constructor( network: NetworkOrSignerOrProvider, address: string, - storage: IStorage, + storage: ThirdwebStorage, options: SDKOptions = {}, abi: typeof ABI, contractWrapper = new ContractWrapper( diff --git a/packages/sdk/src/contracts/prebuilt-implementations/token.ts b/packages/sdk/src/contracts/prebuilt-implementations/token.ts index 4c2896226be..a812425d344 100644 --- a/packages/sdk/src/contracts/prebuilt-implementations/token.ts +++ b/packages/sdk/src/contracts/prebuilt-implementations/token.ts @@ -18,7 +18,7 @@ import { TokenMintInput } from "../../schema/tokens/token"; import { Amount, CurrencyValue } from "../../types"; import type { TokenERC20 } from "@thirdweb-dev/contracts-js"; import type ABI from "@thirdweb-dev/contracts-js/dist/abis/TokenERC20.json"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { CallOverrides, constants } from "ethers"; /** @@ -75,7 +75,7 @@ export class TokenImpl extends StandardErc20 { constructor( network: NetworkOrSignerOrProvider, address: string, - storage: IStorage, + storage: ThirdwebStorage, options: SDKOptions = {}, abi: typeof ABI, contractWrapper = new ContractWrapper( diff --git a/packages/sdk/src/contracts/prebuilt-implementations/vote.ts b/packages/sdk/src/contracts/prebuilt-implementations/vote.ts index 0a173407f9d..66c3339fd7a 100644 --- a/packages/sdk/src/contracts/prebuilt-implementations/vote.ts +++ b/packages/sdk/src/contracts/prebuilt-implementations/vote.ts @@ -28,7 +28,7 @@ import type { IERC20, VoteERC20 } from "@thirdweb-dev/contracts-js"; import ERC20Abi from "@thirdweb-dev/contracts-js/dist/abis/IERC20.json"; import type ABI from "@thirdweb-dev/contracts-js/dist/abis/VoteERC20.json"; import { ProposalCreatedEvent } from "@thirdweb-dev/contracts-js/dist/declarations/src/VoteERC20"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, @@ -53,7 +53,7 @@ import { */ export class VoteImpl implements UpdateableNetwork { private contractWrapper: ContractWrapper; - private storage: IStorage; + private storage: ThirdwebStorage; public abi: typeof ABI; public metadata: ContractMetadata; @@ -68,7 +68,7 @@ export class VoteImpl implements UpdateableNetwork { constructor( network: NetworkOrSignerOrProvider, address: string, - storage: IStorage, + storage: ThirdwebStorage, options: SDKOptions = {}, abi: typeof ABI, contractWrapper = new ContractWrapper( diff --git a/packages/sdk/src/contracts/smart-contract.ts b/packages/sdk/src/contracts/smart-contract.ts index b9e5c6cac23..e2f1c4ecc5f 100644 --- a/packages/sdk/src/contracts/smart-contract.ts +++ b/packages/sdk/src/contracts/smart-contract.ts @@ -38,7 +38,7 @@ import type { IRoyalty, Ownable, } from "@thirdweb-dev/contracts-js"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BaseContract, CallOverrides, ContractInterface } from "ethers"; /** @@ -142,7 +142,7 @@ export class SmartContract network: NetworkOrSignerOrProvider, address: string, abi: ContractInterface, - storage: IStorage, + storage: ThirdwebStorage, options: SDKOptions = {}, contractWrapper = new ContractWrapper( network, diff --git a/packages/sdk/src/core/classes/contract-deployer.ts b/packages/sdk/src/core/classes/contract-deployer.ts index 8c35512a69b..d348c9f7522 100644 --- a/packages/sdk/src/core/classes/contract-deployer.ts +++ b/packages/sdk/src/core/classes/contract-deployer.ts @@ -39,7 +39,7 @@ import { import { ContractFactory } from "./factory"; import { ContractRegistry } from "./registry"; import { RPCConnectionHandler } from "./rpc-connection-handler"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BytesLike, ContractInterface, ethers } from "ethers"; import invariant from "tiny-invariant"; import { z } from "zod"; @@ -59,12 +59,12 @@ export class ContractDeployer extends RPCConnectionHandler { * should never be accessed directly, use {@link ContractDeployer.getRegistry} instead */ private _registry: Promise | undefined; - private storage: IStorage; + private storage: ThirdwebStorage; constructor( network: NetworkOrSignerOrProvider, options: SDKOptions, - storage: IStorage, + storage: ThirdwebStorage, ) { super(network, options); this.storage = storage; diff --git a/packages/sdk/src/core/classes/contract-metadata.ts b/packages/sdk/src/core/classes/contract-metadata.ts index 213d458794d..7a5d6625022 100644 --- a/packages/sdk/src/core/classes/contract-metadata.ts +++ b/packages/sdk/src/core/classes/contract-metadata.ts @@ -12,7 +12,7 @@ import type { IContractMetadata, IERC20Metadata, } from "@thirdweb-dev/contracts-js"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BaseContract } from "ethers"; import { z } from "zod"; @@ -52,7 +52,7 @@ export class ContractMetadata< constructor( contractWrapper: ContractWrapper, schema: TSchema, - storage: IStorage, + storage: ThirdwebStorage, ) { this.contractWrapper = contractWrapper; this.schema = schema; @@ -87,7 +87,7 @@ export class ContractMetadata< if (this.supportsContractMetadata(this.contractWrapper)) { const uri = await this.contractWrapper.readContract.contractURI(); if (uri && uri.includes("://")) { - data = await this.storage.get(uri); + data = await this.storage.downloadJSON(uri); } } @@ -175,7 +175,7 @@ export class ContractMetadata< */ public async _parseAndUploadMetadata(metadata: z.input) { const parsedMetadata = this.parseInputMetadata(metadata); - return this.storage.uploadMetadata(parsedMetadata); + return this.storage.upload(parsedMetadata); } private supportsContractMetadata( diff --git a/packages/sdk/src/core/classes/contract-published-metadata.ts b/packages/sdk/src/core/classes/contract-published-metadata.ts index 41849b03e92..e4f91ccfda1 100644 --- a/packages/sdk/src/core/classes/contract-published-metadata.ts +++ b/packages/sdk/src/core/classes/contract-published-metadata.ts @@ -10,7 +10,7 @@ import { PublishedMetadata, } from "../../schema/contracts/custom"; import { ContractWrapper } from "./contract-wrapper"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BaseContract } from "ethers"; /** @@ -19,11 +19,14 @@ import { BaseContract } from "ethers"; */ export class ContractPublishedMetadata { private contractWrapper; - private storage: IStorage; + private storage: ThirdwebStorage; private _cachedMetadata: PublishedMetadata | undefined; - constructor(contractWrapper: ContractWrapper, storage: IStorage) { + constructor( + contractWrapper: ContractWrapper, + storage: ThirdwebStorage, + ) { this.contractWrapper = contractWrapper; this.storage = storage; } diff --git a/packages/sdk/src/core/classes/contract-publisher.ts b/packages/sdk/src/core/classes/contract-publisher.ts index 3276085169a..c117c09e9f9 100644 --- a/packages/sdk/src/core/classes/contract-publisher.ts +++ b/packages/sdk/src/core/classes/contract-publisher.ts @@ -35,7 +35,7 @@ import type { } from "@thirdweb-dev/contracts-js"; import ContractPublisherAbi from "@thirdweb-dev/contracts-js/dist/abis/ContractPublisher.json"; import { ContractPublishedEvent } from "@thirdweb-dev/contracts-js/dist/declarations/src/ContractPublisher"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { constants, utils } from "ethers"; import invariant from "tiny-invariant"; @@ -44,13 +44,13 @@ import invariant from "tiny-invariant"; * @internal */ export class ContractPublisher extends RPCConnectionHandler { - private storage: IStorage; + private storage: ThirdwebStorage; private publisher: ContractWrapper; constructor( network: NetworkOrSignerOrProvider, options: SDKOptions, - storage: IStorage, + storage: ThirdwebStorage, ) { super(network, options); this.storage = storage; @@ -225,7 +225,7 @@ export class ContractPublisher extends RPCConnectionHandler { const signer = this.getSigner(); invariant(signer, "A signer is required"); const publisher = await signer.getAddress(); - const profileUri = await this.storage.uploadMetadata(profileMetadata); + const profileUri = await this.storage.upload(profileMetadata); return { receipt: await this.publisher.sendTransaction("setPublisherProfileUri", [ publisher, @@ -247,7 +247,9 @@ export class ContractPublisher extends RPCConnectionHandler { if (!profileUri || profileUri.length === 0) { return {}; } - return ProfileSchemaOutput.parse(await this.storage.get(profileUri)); + return ProfileSchemaOutput.parse( + await this.storage.downloadJSON(profileUri), + ); } /** @@ -335,9 +337,9 @@ export class ContractPublisher extends RPCConnectionHandler { } } - const fetchedBytecode = await this.storage.getRaw( - predeployMetadata.bytecodeUri, - ); + const fetchedBytecode = await ( + await this.storage.download(predeployMetadata.bytecodeUri) + ).text(); const bytecode = fetchedBytecode.startsWith("0x") ? fetchedBytecode : `0x${fetchedBytecode}`; @@ -353,7 +355,7 @@ export class ContractPublisher extends RPCConnectionHandler { analytics: predeployMetadata.analytics, publisher, }); - const fullMetadataUri = await this.storage.uploadMetadata(fullMetadata); + const fullMetadataUri = await this.storage.upload(fullMetadata); const receipt = await this.publisher.sendTransaction("publishContract", [ publisher, contractId, diff --git a/packages/sdk/src/core/classes/delayed-reveal.ts b/packages/sdk/src/core/classes/delayed-reveal.ts index 71313c505a2..77de104113f 100644 --- a/packages/sdk/src/core/classes/delayed-reveal.ts +++ b/packages/sdk/src/core/classes/delayed-reveal.ts @@ -1,5 +1,8 @@ import { hasFunction } from "../../common"; -import { fetchTokenMetadataForContract } from "../../common/nft"; +import { + fetchTokenMetadataForContract, + getBaseUriFromBatch, +} from "../../common/nft"; import { FeatureName } from "../../constants/contract-features"; import { CommonNFTInput, @@ -21,7 +24,7 @@ import type { } from "@thirdweb-dev/contracts-js"; import DeprecatedAbi from "@thirdweb-dev/contracts-js/dist/abis/IDelayedRevealDeprecated.json"; import { TokensLazyMintedEvent } from "@thirdweb-dev/contracts-js/dist/declarations/src/DropERC721"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, ethers } from "ethers"; /** @@ -38,12 +41,12 @@ export class DelayedReveal< featureName; private contractWrapper: ContractWrapper; - private storage: IStorage; + private storage: ThirdwebStorage; private nextTokenIdToMintFn: () => Promise; constructor( contractWrapper: ContractWrapper, - storage: IStorage, + storage: ThirdwebStorage, fetureName: FeatureName, nextTokenIdToMintFn: () => Promise, ) { @@ -98,26 +101,29 @@ export class DelayedReveal< throw new Error("Password is required"); } - const { baseUri: placeholderUri } = await this.storage.uploadMetadataBatch( + const placeholderUris = await this.storage.uploadBatch( [CommonNFTInput.parse(placeholder)], - 0, - this.contractWrapper.readContract.address, - await this.contractWrapper.getSigner()?.getAddress(), + { + rewriteFileNames: { + fileStartNumber: 0, + }, + }, ); + const placeholderUri = getBaseUriFromBatch(placeholderUris); const startFileNumber = await this.nextTokenIdToMintFn(); - const batch = await this.storage.uploadMetadataBatch( + const uris = await this.storage.uploadBatch( metadatas.map((m) => CommonNFTInput.parse(m)), - startFileNumber.toNumber(), - this.contractWrapper.readContract.address, - await this.contractWrapper.getSigner()?.getAddress(), - options, + { + onProgress: options?.onProgress, + rewriteFileNames: { + fileStartNumber: startFileNumber.toNumber(), + }, + }, ); - const baseUri = batch.baseUri.endsWith("/") - ? batch.baseUri - : `${batch.baseUri}/`; + const baseUri = getBaseUriFromBatch(uris); const baseUriId = await this.contractWrapper.readContract.getBaseURICount(); const hashedPassword = await this.hashDelayRevealPasword( baseUriId, @@ -146,7 +152,7 @@ export class DelayedReveal< } const receipt = await this.contractWrapper.sendTransaction("lazyMint", [ - batch.uris.length, + uris.length, placeholderUri.endsWith("/") ? placeholderUri : `${placeholderUri}/`, data, ]); diff --git a/packages/sdk/src/core/classes/drop-claim-conditions.ts b/packages/sdk/src/core/classes/drop-claim-conditions.ts index 716f55be8b2..f41b1f300cd 100644 --- a/packages/sdk/src/core/classes/drop-claim-conditions.ts +++ b/packages/sdk/src/core/classes/drop-claim-conditions.ts @@ -36,7 +36,7 @@ import type { } from "@thirdweb-dev/contracts-js"; import ERC20Abi from "@thirdweb-dev/contracts-js/dist/abis/IERC20.json"; import { IDropClaimCondition } from "@thirdweb-dev/contracts-js/dist/declarations/src/DropERC20"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, constants, ethers, utils } from "ethers"; import deepEqual from "fast-deep-equal"; @@ -54,12 +54,12 @@ export class DropClaimConditions< > { private contractWrapper; private metadata; - private storage: IStorage; + private storage: ThirdwebStorage; constructor( contractWrapper: ContractWrapper, metadata: ContractMetadata, - storage: IStorage, + storage: ThirdwebStorage, ) { this.storage = storage; this.contractWrapper = contractWrapper; diff --git a/packages/sdk/src/core/classes/drop-erc1155-claim-conditions.ts b/packages/sdk/src/core/classes/drop-erc1155-claim-conditions.ts index 52b72dec90b..de3cf4e54e8 100644 --- a/packages/sdk/src/core/classes/drop-erc1155-claim-conditions.ts +++ b/packages/sdk/src/core/classes/drop-erc1155-claim-conditions.ts @@ -29,7 +29,7 @@ import type { } from "@thirdweb-dev/contracts-js"; import IERC20ABI from "@thirdweb-dev/contracts-js/dist/abis/IERC20.json"; import { IDropClaimCondition } from "@thirdweb-dev/contracts-js/dist/declarations/src/DropERC1155"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, constants, ethers, utils } from "ethers"; import deepEqual from "fast-deep-equal"; @@ -42,12 +42,12 @@ export class DropErc1155ClaimConditions< > { private contractWrapper; private metadata; - private storage: IStorage; + private storage: ThirdwebStorage; constructor( contractWrapper: ContractWrapper, metadata: ContractMetadata, - storage: IStorage, + storage: ThirdwebStorage, ) { this.storage = storage; this.contractWrapper = contractWrapper; diff --git a/packages/sdk/src/core/classes/erc-1155-batch-mintable.ts b/packages/sdk/src/core/classes/erc-1155-batch-mintable.ts index a4aacbb6818..04edd65a6e6 100644 --- a/packages/sdk/src/core/classes/erc-1155-batch-mintable.ts +++ b/packages/sdk/src/core/classes/erc-1155-batch-mintable.ts @@ -7,7 +7,7 @@ import { ContractWrapper } from "./contract-wrapper"; import { Erc1155 } from "./erc-1155"; import type { IMintableERC1155, IMulticall } from "@thirdweb-dev/contracts-js"; import { TokensMintedEvent } from "@thirdweb-dev/contracts-js/dist/declarations/src/TokenERC1155"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { ethers } from "ethers"; /** @@ -24,12 +24,12 @@ export class Erc1155BatchMintable implements DetectableFeature { featureName = FEATURE_EDITION_BATCH_MINTABLE.name; private contractWrapper: ContractWrapper; private erc1155: Erc1155; - private storage: IStorage; + private storage: ThirdwebStorage; constructor( erc1155: Erc1155, contractWrapper: ContractWrapper, - storage: IStorage, + storage: ThirdwebStorage, ) { this.erc1155 = erc1155; this.contractWrapper = contractWrapper; diff --git a/packages/sdk/src/core/classes/erc-1155-claimable-with-conditions.ts b/packages/sdk/src/core/classes/erc-1155-claimable-with-conditions.ts index a0a42075f8c..14d062b4d7f 100644 --- a/packages/sdk/src/core/classes/erc-1155-claimable-with-conditions.ts +++ b/packages/sdk/src/core/classes/erc-1155-claimable-with-conditions.ts @@ -13,7 +13,7 @@ import { ContractMetadata } from "./contract-metadata"; import { ContractWrapper } from "./contract-wrapper"; import { DropErc1155ClaimConditions } from "./drop-erc1155-claim-conditions"; import { DropERC1155 } from "@thirdweb-dev/contracts-js"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumberish, ethers } from "ethers"; /** @@ -31,11 +31,11 @@ export class Erc1155ClaimableWithConditions implements DetectableFeature { public conditions: DropErc1155ClaimConditions; private contractWrapper: ContractWrapper; - private storage: IStorage; + private storage: ThirdwebStorage; constructor( contractWrapper: ContractWrapper, - storage: IStorage, + storage: ThirdwebStorage, ) { this.contractWrapper = contractWrapper; this.storage = storage; diff --git a/packages/sdk/src/core/classes/erc-1155-lazymintable.ts b/packages/sdk/src/core/classes/erc-1155-lazymintable.ts index 7b5eb5d7ba6..55759630010 100644 --- a/packages/sdk/src/core/classes/erc-1155-lazymintable.ts +++ b/packages/sdk/src/core/classes/erc-1155-lazymintable.ts @@ -26,7 +26,7 @@ import type { TokenERC721, } from "@thirdweb-dev/contracts-js"; import { TokensLazyMintedEvent } from "@thirdweb-dev/contracts-js/dist/declarations/src/LazyMint"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { ethers } from "ethers"; export class Erc1155LazyMintable implements DetectableFeature { @@ -80,12 +80,12 @@ export class Erc1155LazyMintable implements DetectableFeature { private contractWrapper: ContractWrapper; private erc1155: Erc1155; - private storage: IStorage; + private storage: ThirdwebStorage; constructor( erc1155: Erc1155, contractWrapper: ContractWrapper, - storage: IStorage, + storage: ThirdwebStorage, ) { this.erc1155 = erc1155; this.contractWrapper = contractWrapper; @@ -133,8 +133,6 @@ export class Erc1155LazyMintable implements DetectableFeature { metadatas, this.storage, startFileNumber.toNumber(), - this.contractWrapper.readContract.address, - await this.contractWrapper.getSigner()?.getAddress(), options, ); // ensure baseUri is the same for the entire batch diff --git a/packages/sdk/src/core/classes/erc-1155-mintable.ts b/packages/sdk/src/core/classes/erc-1155-mintable.ts index fe16153ebb3..fccefc3455d 100644 --- a/packages/sdk/src/core/classes/erc-1155-mintable.ts +++ b/packages/sdk/src/core/classes/erc-1155-mintable.ts @@ -9,7 +9,7 @@ import { Erc1155 } from "./erc-1155"; import { Erc1155BatchMintable } from "./erc-1155-batch-mintable"; import type { IMintableERC1155, IMulticall } from "@thirdweb-dev/contracts-js"; import { TransferSingleEvent } from "@thirdweb-dev/contracts-js/dist/declarations/src/ITokenERC1155"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, ethers } from "ethers"; /** @@ -26,7 +26,7 @@ export class Erc1155Mintable implements DetectableFeature { featureName = FEATURE_EDITION_MINTABLE.name; private contractWrapper: ContractWrapper; private erc1155: Erc1155; - private storage: IStorage; + private storage: ThirdwebStorage; /** * Batch mint Tokens to many addresses @@ -36,7 +36,7 @@ export class Erc1155Mintable implements DetectableFeature { constructor( erc1155: Erc1155, contractWrapper: ContractWrapper, - storage: IStorage, + storage: ThirdwebStorage, ) { this.erc1155 = erc1155; this.contractWrapper = contractWrapper; diff --git a/packages/sdk/src/core/classes/erc-1155-signature-mintable.ts b/packages/sdk/src/core/classes/erc-1155-signature-mintable.ts index 46cacf4f2d8..f0d79222701 100644 --- a/packages/sdk/src/core/classes/erc-1155-signature-mintable.ts +++ b/packages/sdk/src/core/classes/erc-1155-signature-mintable.ts @@ -19,7 +19,7 @@ import { ContractRoles } from "./contract-roles"; import { ContractWrapper } from "./contract-wrapper"; import type { ITokenERC1155, TokenERC1155 } from "@thirdweb-dev/contracts-js"; import { TokensMintedWithSignatureEvent } from "@thirdweb-dev/contracts-js/dist/declarations/src/ITokenERC1155"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, ethers } from "ethers"; import invariant from "tiny-invariant"; @@ -33,14 +33,14 @@ export class Erc1155SignatureMintable implements DetectableFeature { private contractWrapper: ContractWrapper< BaseSignatureMintERC1155 | TokenERC1155 >; - private storage: IStorage; + private storage: ThirdwebStorage; private roles: | ContractRoles | undefined; constructor( contractWrapper: ContractWrapper, - storage: IStorage, + storage: ThirdwebStorage, roles?: ContractRoles, ) { this.contractWrapper = contractWrapper; diff --git a/packages/sdk/src/core/classes/erc-1155-standard.ts b/packages/sdk/src/core/classes/erc-1155-standard.ts index 9fd5bdee637..52bf5dd4253 100644 --- a/packages/sdk/src/core/classes/erc-1155-standard.ts +++ b/packages/sdk/src/core/classes/erc-1155-standard.ts @@ -6,7 +6,7 @@ import { NetworkOrSignerOrProvider, TransactionResult } from "../types"; import { ContractWrapper } from "./contract-wrapper"; import { Erc1155 } from "./erc-1155"; import type { DropERC1155, TokenERC1155 } from "@thirdweb-dev/contracts-js"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, BytesLike } from "ethers"; /** @@ -26,10 +26,10 @@ export class StandardErc1155< > implements UpdateableNetwork { protected contractWrapper: ContractWrapper; - protected storage: IStorage; + protected storage: ThirdwebStorage; public erc1155: Erc1155; - constructor(contractWrapper: ContractWrapper, storage: IStorage) { + constructor(contractWrapper: ContractWrapper, storage: ThirdwebStorage) { this.contractWrapper = contractWrapper; this.storage = storage; this.erc1155 = new Erc1155(this.contractWrapper, this.storage); diff --git a/packages/sdk/src/core/classes/erc-1155.ts b/packages/sdk/src/core/classes/erc-1155.ts index bb8a8176347..add8704896d 100644 --- a/packages/sdk/src/core/classes/erc-1155.ts +++ b/packages/sdk/src/core/classes/erc-1155.ts @@ -54,7 +54,7 @@ import type { IMintableERC1155, TokenERC1155, } from "@thirdweb-dev/contracts-js"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, BytesLike } from "ethers"; /** @@ -81,9 +81,9 @@ export class Erc1155< private signatureMintable: Erc1155SignatureMintable | undefined; protected contractWrapper: ContractWrapper; - protected storage: IStorage; + protected storage: ThirdwebStorage; - constructor(contractWrapper: ContractWrapper, storage: IStorage) { + constructor(contractWrapper: ContractWrapper, storage: ThirdwebStorage) { this.contractWrapper = contractWrapper; this.storage = storage; this.query = this.detectErc1155Enumerable(); diff --git a/packages/sdk/src/core/classes/erc-20-claimable.ts b/packages/sdk/src/core/classes/erc-20-claimable.ts index 4f3437bd4b4..773390e7261 100644 --- a/packages/sdk/src/core/classes/erc-20-claimable.ts +++ b/packages/sdk/src/core/classes/erc-20-claimable.ts @@ -9,7 +9,7 @@ import { ContractMetadata } from "./contract-metadata"; import { ContractWrapper } from "./contract-wrapper"; import { DropClaimConditions } from "./drop-claim-conditions"; import { Erc20 } from "./erc-20"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; /** * Configure and claim ERC20 tokens @@ -47,12 +47,12 @@ export class Erc20Claimable implements DetectableFeature { public conditions: DropClaimConditions; private contractWrapper: ContractWrapper; private erc20: Erc20; - private storage: IStorage; + private storage: ThirdwebStorage; constructor( erc20: Erc20, contractWrapper: ContractWrapper, - storage: IStorage, + storage: ThirdwebStorage, ) { this.erc20 = erc20; this.contractWrapper = contractWrapper; diff --git a/packages/sdk/src/core/classes/erc-20-droppable.ts b/packages/sdk/src/core/classes/erc-20-droppable.ts index 82aaa333608..5fe81cabe9f 100644 --- a/packages/sdk/src/core/classes/erc-20-droppable.ts +++ b/packages/sdk/src/core/classes/erc-20-droppable.ts @@ -2,7 +2,7 @@ import { BaseDropERC20 } from "../../types/eips"; import { ContractWrapper } from "./contract-wrapper"; import { Erc20 } from "./erc-20"; import { Erc20Claimable } from "./erc-20-claimable"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; /** * Configure and claim ERC20 tokens @@ -39,12 +39,12 @@ export class Erc20Droppable { public claim: Erc20Claimable; private contractWrapper: ContractWrapper; private erc20: Erc20; - private storage: IStorage; + private storage: ThirdwebStorage; constructor( erc20: Erc20, contractWrapper: ContractWrapper, - storage: IStorage, + storage: ThirdwebStorage, ) { this.erc20 = erc20; this.contractWrapper = contractWrapper; diff --git a/packages/sdk/src/core/classes/erc-20-standard.ts b/packages/sdk/src/core/classes/erc-20-standard.ts index fc6cc09f4e7..ec3692cb517 100644 --- a/packages/sdk/src/core/classes/erc-20-standard.ts +++ b/packages/sdk/src/core/classes/erc-20-standard.ts @@ -3,11 +3,10 @@ import { Amount, Currency, CurrencyValue } from "../../types/currency"; import { BaseERC20, BaseSignatureMintERC20 } from "../../types/eips"; import { UpdateableNetwork } from "../interfaces/contract"; import { NetworkOrSignerOrProvider, TransactionResult } from "../types"; -import { ContractPrimarySale } from "./contract-sales"; import { ContractWrapper } from "./contract-wrapper"; import { Erc20 } from "./erc-20"; import type { DropERC20, TokenERC20 } from "@thirdweb-dev/contracts-js"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; /** * Standard ERC20 Token functions @@ -26,11 +25,11 @@ export class StandardErc20< > implements UpdateableNetwork { protected contractWrapper: ContractWrapper; - protected storage: IStorage; + protected storage: ThirdwebStorage; public erc20: Erc20; - constructor(contractWrapper: ContractWrapper, storage: IStorage) { + constructor(contractWrapper: ContractWrapper, storage: ThirdwebStorage) { this.contractWrapper = contractWrapper; this.storage = storage; this.erc20 = new Erc20(this.contractWrapper, this.storage); diff --git a/packages/sdk/src/core/classes/erc-20.ts b/packages/sdk/src/core/classes/erc-20.ts index a882d5048a6..ed52193d7ae 100644 --- a/packages/sdk/src/core/classes/erc-20.ts +++ b/packages/sdk/src/core/classes/erc-20.ts @@ -37,7 +37,7 @@ import type { IMintableERC20, IBurnableERC20, } from "@thirdweb-dev/contracts-js"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { ethers, BigNumber, BigNumberish } from "ethers"; /** @@ -65,9 +65,9 @@ export class Erc20< private droppable: Erc20Droppable | undefined; private signatureMintable: Erc20SignatureMintable | undefined; protected contractWrapper: ContractWrapper; - protected storage: IStorage; + protected storage: ThirdwebStorage; - constructor(contractWrapper: ContractWrapper, storage: IStorage) { + constructor(contractWrapper: ContractWrapper, storage: ThirdwebStorage) { this.contractWrapper = contractWrapper; this.storage = storage; this.mintable = this.detectErc20Mintable(); diff --git a/packages/sdk/src/core/classes/erc-721-batch-mintable.ts b/packages/sdk/src/core/classes/erc-721-batch-mintable.ts index 64d31804d0e..178d8830306 100644 --- a/packages/sdk/src/core/classes/erc-721-batch-mintable.ts +++ b/packages/sdk/src/core/classes/erc-721-batch-mintable.ts @@ -7,7 +7,7 @@ import { ContractWrapper } from "./contract-wrapper"; import { Erc721 } from "./erc-721"; import type { IMintableERC721, IMulticall } from "@thirdweb-dev/contracts-js"; import { TokensMintedEvent } from "@thirdweb-dev/contracts-js/dist/declarations/src/IMintableERC721"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; /** * Mint Many ERC721 NFTs at once @@ -22,13 +22,13 @@ import { IStorage } from "@thirdweb-dev/storage"; export class Erc721BatchMintable implements DetectableFeature { featureName = FEATURE_NFT_BATCH_MINTABLE.name; private contractWrapper: ContractWrapper; - private storage: IStorage; + private storage: ThirdwebStorage; private erc721: Erc721; constructor( erc721: Erc721, contractWrapper: ContractWrapper, - storage: IStorage, + storage: ThirdwebStorage, ) { this.erc721 = erc721; this.contractWrapper = contractWrapper; diff --git a/packages/sdk/src/core/classes/erc-721-claimable-with-conditions.ts b/packages/sdk/src/core/classes/erc-721-claimable-with-conditions.ts index 9ff5e04f5d4..c785d50f87f 100644 --- a/packages/sdk/src/core/classes/erc-721-claimable-with-conditions.ts +++ b/packages/sdk/src/core/classes/erc-721-claimable-with-conditions.ts @@ -13,7 +13,7 @@ import { DropClaimConditions } from "./drop-claim-conditions"; import { Erc721 } from "./erc-721"; import type { DropERC721 } from "@thirdweb-dev/contracts-js"; import { TokensClaimedEvent } from "@thirdweb-dev/contracts-js/dist/declarations/src/Drop"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, ethers } from "ethers"; /** @@ -54,12 +54,12 @@ export class Erc721ClaimableWithConditions implements DetectableFeature { public conditions: DropClaimConditions; private contractWrapper: ContractWrapper; private erc721: Erc721; - private storage: IStorage; + private storage: ThirdwebStorage; constructor( erc721: Erc721, contractWrapper: ContractWrapper, - storage: IStorage, + storage: ThirdwebStorage, ) { this.erc721 = erc721; this.contractWrapper = contractWrapper; diff --git a/packages/sdk/src/core/classes/erc-721-lazymintable.ts b/packages/sdk/src/core/classes/erc-721-lazymintable.ts index 5785f8df0c7..3d94d6fdb90 100644 --- a/packages/sdk/src/core/classes/erc-721-lazymintable.ts +++ b/packages/sdk/src/core/classes/erc-721-lazymintable.ts @@ -2,7 +2,7 @@ import { detectContractFeature, hasFunction, } from "../../common/feature-detection"; -import { uploadOrExtractURIs } from "../../common/nft"; +import { getBaseUriFromBatch, uploadOrExtractURIs } from "../../common/nft"; import { FEATURE_NFT_LAZY_MINTABLE, FEATURE_NFT_REVEALABLE, @@ -23,7 +23,7 @@ import { Erc721Claimable } from "./erc-721-claimable"; import { Erc721ClaimableWithConditions } from "./erc-721-claimable-with-conditions"; import type { IClaimableERC721 } from "@thirdweb-dev/contracts-js"; import { TokensLazyMintedEvent } from "@thirdweb-dev/contracts-js/dist/declarations/src/LazyMint"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { ethers } from "ethers"; /** @@ -85,12 +85,12 @@ export class Erc721LazyMintable implements DetectableFeature { private contractWrapper: ContractWrapper; private erc721: Erc721; - private storage: IStorage; + private storage: ThirdwebStorage; constructor( erc721: Erc721, contractWrapper: ContractWrapper, - storage: IStorage, + storage: ThirdwebStorage, ) { this.erc721 = erc721; this.contractWrapper = contractWrapper; @@ -138,20 +138,11 @@ export class Erc721LazyMintable implements DetectableFeature { metadatas, this.storage, startFileNumber.toNumber(), - this.contractWrapper.readContract.address, - await this.contractWrapper.getSigner()?.getAddress(), options, ); // ensure baseUri is the same for the entire batch - const baseUri = batch[0].substring(0, batch[0].lastIndexOf("/")); - for (let i = 0; i < batch.length; i++) { - const uri = batch[i].substring(0, batch[i].lastIndexOf("/")); - if (baseUri !== uri) { - throw new Error( - `Can only create batches with the same base URI for every entry in the batch. Expected '${baseUri}' but got '${uri}'`, - ); - } - } + const baseUri = getBaseUriFromBatch(batch); + const receipt = await this.contractWrapper.sendTransaction("lazyMint", [ batch.length, baseUri.endsWith("/") ? baseUri : `${baseUri}/`, diff --git a/packages/sdk/src/core/classes/erc-721-mintable.ts b/packages/sdk/src/core/classes/erc-721-mintable.ts index 92f284bdc8d..d523b252f23 100644 --- a/packages/sdk/src/core/classes/erc-721-mintable.ts +++ b/packages/sdk/src/core/classes/erc-721-mintable.ts @@ -9,7 +9,7 @@ import { Erc721 } from "./erc-721"; import { Erc721BatchMintable } from "./erc-721-batch-mintable"; import type { IMintableERC721, IMulticall } from "@thirdweb-dev/contracts-js"; import { TransferEvent } from "@thirdweb-dev/contracts-js/dist/declarations/src/ITokenERC721"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; /** * Mint ERC721 NFTs @@ -24,7 +24,7 @@ import { IStorage } from "@thirdweb-dev/storage"; export class Erc721Mintable implements DetectableFeature { featureName = FEATURE_NFT_MINTABLE.name; private contractWrapper: ContractWrapper; - private storage: IStorage; + private storage: ThirdwebStorage; private erc721: Erc721; public batch: Erc721BatchMintable | undefined; @@ -32,7 +32,7 @@ export class Erc721Mintable implements DetectableFeature { constructor( erc721: Erc721, contractWrapper: ContractWrapper, - storage: IStorage, + storage: ThirdwebStorage, ) { this.erc721 = erc721; this.contractWrapper = contractWrapper; diff --git a/packages/sdk/src/core/classes/erc-721-standard.ts b/packages/sdk/src/core/classes/erc-721-standard.ts index 2b36302a5fe..0cff67eec6e 100644 --- a/packages/sdk/src/core/classes/erc-721-standard.ts +++ b/packages/sdk/src/core/classes/erc-721-standard.ts @@ -10,7 +10,7 @@ import type { SignatureDrop, TokenERC721, } from "@thirdweb-dev/contracts-js"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish } from "ethers"; /** @@ -28,10 +28,10 @@ export class StandardErc721< > implements UpdateableNetwork { protected contractWrapper: ContractWrapper; - protected storage: IStorage; + protected storage: ThirdwebStorage; public erc721: Erc721; - constructor(contractWrapper: ContractWrapper, storage: IStorage) { + constructor(contractWrapper: ContractWrapper, storage: ThirdwebStorage) { this.contractWrapper = contractWrapper; this.storage = storage; this.erc721 = new Erc721(this.contractWrapper, this.storage); diff --git a/packages/sdk/src/core/classes/erc-721-with-quantity-signature-mintable.ts b/packages/sdk/src/core/classes/erc-721-with-quantity-signature-mintable.ts index 3ff014967dd..89d97a5bc8b 100644 --- a/packages/sdk/src/core/classes/erc-721-with-quantity-signature-mintable.ts +++ b/packages/sdk/src/core/classes/erc-721-with-quantity-signature-mintable.ts @@ -21,7 +21,7 @@ import type { TokenERC721, } from "@thirdweb-dev/contracts-js"; import { TokensMintedWithSignatureEvent } from "@thirdweb-dev/contracts-js/dist/declarations/src/SignatureDrop"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, ethers } from "ethers"; import invariant from "tiny-invariant"; @@ -34,11 +34,11 @@ export class Erc721WithQuantitySignatureMintable implements DetectableFeature { private contractWrapper: ContractWrapper; - private storage: IStorage; + private storage: ThirdwebStorage; constructor( contractWrapper: ContractWrapper, - storage: IStorage, + storage: ThirdwebStorage, ) { this.contractWrapper = contractWrapper; this.storage = storage; diff --git a/packages/sdk/src/core/classes/erc-721.ts b/packages/sdk/src/core/classes/erc-721.ts index 55a87a1b069..4a3b2af8c88 100644 --- a/packages/sdk/src/core/classes/erc-721.ts +++ b/packages/sdk/src/core/classes/erc-721.ts @@ -52,7 +52,7 @@ import type { SignatureDrop, TokenERC721, } from "@thirdweb-dev/contracts-js"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, constants } from "ethers"; /** @@ -81,9 +81,9 @@ export class Erc721< private lazyMintable: Erc721LazyMintable | undefined; private signatureMintable: Erc721WithQuantitySignatureMintable | undefined; protected contractWrapper: ContractWrapper; - protected storage: IStorage; + protected storage: ThirdwebStorage; - constructor(contractWrapper: ContractWrapper, storage: IStorage) { + constructor(contractWrapper: ContractWrapper, storage: ThirdwebStorage) { this.contractWrapper = contractWrapper; this.storage = storage; this.query = this.detectErc721Enumerable(); diff --git a/packages/sdk/src/core/classes/factory.ts b/packages/sdk/src/core/classes/factory.ts index 41970a15638..88bc1317257 100644 --- a/packages/sdk/src/core/classes/factory.ts +++ b/packages/sdk/src/core/classes/factory.ts @@ -25,7 +25,7 @@ import { ContractWrapper } from "./contract-wrapper"; import type { TWFactory } from "@thirdweb-dev/contracts-js"; import TWFactoryAbi from "@thirdweb-dev/contracts-js/dist/abis/TWFactory.json"; import { ProxyDeployedEvent } from "@thirdweb-dev/contracts-js/dist/declarations/src/TWFactory"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, constants, @@ -39,12 +39,12 @@ import { z } from "zod"; * @internal */ export class ContractFactory extends ContractWrapper { - private storage: IStorage; + private storage: ThirdwebStorage; constructor( factoryAddr: string, network: NetworkOrSignerOrProvider, - storage: IStorage, + storage: ThirdwebStorage, options?: SDKOptions, ) { super(network, factoryAddr, TWFactoryAbi, options); @@ -61,11 +61,7 @@ export class ContractFactory extends ContractWrapper { const metadata = contract.schema.deploy.parse(contractMetadata); // TODO: is there any special pre-processing we need to do before uploading? - const contractURI = await this.storage.uploadMetadata( - metadata, - this.readContract.address, - await this.getSigner()?.getAddress(), - ); + const contractURI = await this.storage.upload(metadata); const ABI = await contract.getAbi(); diff --git a/packages/sdk/src/core/classes/marketplace-auction.ts b/packages/sdk/src/core/classes/marketplace-auction.ts index 40bb7b679f8..a3767555c9e 100644 --- a/packages/sdk/src/core/classes/marketplace-auction.ts +++ b/packages/sdk/src/core/classes/marketplace-auction.ts @@ -28,7 +28,7 @@ import { TransactionResult, TransactionResultWithId } from "../types"; import { ContractWrapper } from "./contract-wrapper"; import type { IMarketplace, Marketplace } from "@thirdweb-dev/contracts-js"; import { ListingAddedEvent } from "@thirdweb-dev/contracts-js/dist/declarations/src/Marketplace"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, ethers, constants } from "ethers"; import invariant from "tiny-invariant"; @@ -38,11 +38,11 @@ import invariant from "tiny-invariant"; */ export class MarketplaceAuction { private contractWrapper: ContractWrapper; - private storage: IStorage; + private storage: ThirdwebStorage; constructor( contractWrapper: ContractWrapper, - storage: IStorage, + storage: ThirdwebStorage, ) { this.contractWrapper = contractWrapper; this.storage = storage; diff --git a/packages/sdk/src/core/classes/marketplace-direct.ts b/packages/sdk/src/core/classes/marketplace-direct.ts index 56fc71f350c..507afa66ec0 100644 --- a/packages/sdk/src/core/classes/marketplace-direct.ts +++ b/packages/sdk/src/core/classes/marketplace-direct.ts @@ -36,7 +36,7 @@ import ERC165Abi from "@thirdweb-dev/contracts-js/dist/abis/IERC165.json"; import ERC721Abi from "@thirdweb-dev/contracts-js/dist/abis/IERC721.json"; import ERC1155Abi from "@thirdweb-dev/contracts-js/dist/abis/IERC1155.json"; import { ListingAddedEvent } from "@thirdweb-dev/contracts-js/dist/declarations/src/Marketplace"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { BigNumber, BigNumberish, @@ -53,11 +53,11 @@ import invariant from "tiny-invariant"; */ export class MarketplaceDirect { private contractWrapper: ContractWrapper; - private storage: IStorage; + private storage: ThirdwebStorage; constructor( contractWrapper: ContractWrapper, - storage: IStorage, + storage: ThirdwebStorage, ) { this.contractWrapper = contractWrapper; this.storage = storage; diff --git a/packages/sdk/src/core/sdk.ts b/packages/sdk/src/core/sdk.ts index cc02a2f39a6..9ceca03e740 100644 --- a/packages/sdk/src/core/sdk.ts +++ b/packages/sdk/src/core/sdk.ts @@ -39,7 +39,7 @@ import type { } from "./types"; import { UserWallet } from "./wallet/UserWallet"; import IThirdwebContractABI from "@thirdweb-dev/contracts-js/dist/abis/IThirdwebContract.json"; -import { IpfsStorage, RemoteStorage, IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { Contract, ContractInterface, ethers, Signer } from "ethers"; import invariant from "tiny-invariant"; @@ -72,7 +72,7 @@ export class ThirdwebSDK extends RPCConnectionHandler { signer: Signer, network?: ChainOrRpc, options: SDKOptions = {}, - storage: IStorage = new IpfsStorage(), + storage: ThirdwebStorage = new ThirdwebStorage(), ): ThirdwebSDK { const sdk = new ThirdwebSDK(network || signer, options, storage); sdk.updateSignerOrProvider(signer); @@ -103,7 +103,7 @@ export class ThirdwebSDK extends RPCConnectionHandler { privateKey: string, network: ChainOrRpc, options: SDKOptions = {}, - storage: IStorage = new IpfsStorage(), + storage: ThirdwebStorage = new ThirdwebStorage(), ): ThirdwebSDK { const signerOrProvider = getProviderForNetwork(network); const provider = Signer.isSigner(signerOrProvider) @@ -128,7 +128,7 @@ export class ThirdwebSDK extends RPCConnectionHandler { /** * Internal handler for uploading and downloading files */ - private storageHandler: IStorage; + private storageHandler: ThirdwebStorage; /** * New contract deployer */ @@ -140,7 +140,7 @@ export class ThirdwebSDK extends RPCConnectionHandler { /** * Upload and download files from IPFS or from your own storage service */ - public storage: RemoteStorage; + public storage: ThirdwebStorage; /** * Enable authentication with the connected wallet */ @@ -149,12 +149,12 @@ export class ThirdwebSDK extends RPCConnectionHandler { constructor( network: ChainOrRpc | SignerOrProvider, options: SDKOptions = {}, - storage: IStorage = new IpfsStorage(), + storage: ThirdwebStorage = new ThirdwebStorage(), ) { const signerOrProvider = getProviderForNetwork(network); super(signerOrProvider, options); this.storageHandler = storage; - this.storage = new RemoteStorage(storage); + this.storage = storage; this.wallet = new UserWallet(signerOrProvider, options); this.deployer = new ContractDeployer(signerOrProvider, options, storage); this.auth = new WalletAuthenticator(signerOrProvider, this.wallet, options); diff --git a/packages/sdk/src/schema/contracts/common/index.ts b/packages/sdk/src/schema/contracts/common/index.ts index 563ebc1bb8c..c47f654353e 100644 --- a/packages/sdk/src/schema/contracts/common/index.ts +++ b/packages/sdk/src/schema/contracts/common/index.ts @@ -1,5 +1,5 @@ import { AddressSchema, BasisPointsSchema, JsonSchema } from "../../shared"; -import { FileBufferOrStringSchema } from "@thirdweb-dev/storage"; +import { FileOrBufferOrStringSchema } from "@thirdweb-dev/storage"; import { constants } from "ethers"; import { z } from "zod"; @@ -9,7 +9,7 @@ import { z } from "zod"; export const CommonContractSchema = z.object({ name: z.string(), description: z.string().optional(), - image: FileBufferOrStringSchema.optional(), + image: FileOrBufferOrStringSchema.optional(), external_link: z.string().url().optional(), }); diff --git a/packages/sdk/src/schema/contracts/custom.ts b/packages/sdk/src/schema/contracts/custom.ts index 706d3c15431..96b144e5399 100644 --- a/packages/sdk/src/schema/contracts/custom.ts +++ b/packages/sdk/src/schema/contracts/custom.ts @@ -10,7 +10,7 @@ import { CommonTrustedForwarderSchema, MerkleSchema, } from "./common"; -import { FileBufferOrStringSchema } from "@thirdweb-dev/storage"; +import { FileOrBufferOrStringSchema } from "@thirdweb-dev/storage"; import { BigNumberish } from "ethers"; import { z } from "zod"; @@ -145,8 +145,8 @@ export const ExtraPublishMetadataSchemaInput = z license: z.string().optional(), changelog: z.string().optional(), tags: z.array(z.string()).optional(), - audit: FileBufferOrStringSchema.nullable().optional(), - logo: FileBufferOrStringSchema.nullable().optional(), + audit: FileOrBufferOrStringSchema.nullable().optional(), + logo: FileOrBufferOrStringSchema.nullable().optional(), isDeployableViaFactory: z.boolean().optional(), factoryDeploymentData: FactoryDeploymentSchema.optional(), }) @@ -190,7 +190,7 @@ export type FullPublishMetadata = z.infer< export const ProfileSchemaInput = z.object({ name: z.string().optional(), bio: z.string().optional(), - avatar: FileBufferOrStringSchema.nullable().optional(), + avatar: FileOrBufferOrStringSchema.nullable().optional(), website: z.string().optional(), twitter: z.string().optional(), telegram: z.string().optional(), diff --git a/packages/sdk/src/schema/tokens/common/index.ts b/packages/sdk/src/schema/tokens/common/index.ts index 984f7b5aa8f..85f0a59c4e2 100644 --- a/packages/sdk/src/schema/tokens/common/index.ts +++ b/packages/sdk/src/schema/tokens/common/index.ts @@ -1,6 +1,6 @@ import { BigNumberSchema, HexColor, JsonSchema } from "../../shared"; import { OptionalPropertiesInput } from "./properties"; -import { FileBufferOrStringSchema } from "@thirdweb-dev/storage"; +import { FileOrBufferOrStringSchema } from "@thirdweb-dev/storage"; import { z } from "zod"; /** @@ -10,8 +10,8 @@ export const CommonTokenInput = z .object({ name: z.union([z.string(), z.number()]).optional(), description: z.string().nullable().optional(), - image: FileBufferOrStringSchema.nullable().optional(), - external_url: FileBufferOrStringSchema.nullable().optional(), + image: FileOrBufferOrStringSchema.nullable().optional(), + external_url: FileOrBufferOrStringSchema.nullable().optional(), }) .catchall(z.lazy(() => JsonSchema)); @@ -29,7 +29,7 @@ export const CommonTokenOutput = CommonTokenInput.extend({ * @internal */ export const CommonNFTInput = CommonTokenInput.extend({ - animation_url: FileBufferOrStringSchema.optional(), + animation_url: FileOrBufferOrStringSchema.optional(), background_color: HexColor.optional(), properties: OptionalPropertiesInput, attributes: OptionalPropertiesInput, diff --git a/packages/sdk/src/types/deploy/deploy-metadata.ts b/packages/sdk/src/types/deploy/deploy-metadata.ts index 51cbec720ca..546c076feff 100644 --- a/packages/sdk/src/types/deploy/deploy-metadata.ts +++ b/packages/sdk/src/types/deploy/deploy-metadata.ts @@ -1,4 +1,4 @@ -import { FileBufferOrString } from "@thirdweb-dev/storage"; +import { FileOrBufferOrString } from "@thirdweb-dev/storage"; import type { BigNumberish, Bytes } from "ethers"; /** @@ -17,7 +17,7 @@ export interface NFTContractDeployMetadata { /** * Optional image for the contract */ - image?: FileBufferOrString; + image?: FileOrBufferOrString; /** * Optional url for the contract */ @@ -68,7 +68,7 @@ export interface TokenContractDeployMetadata { /** * Optional image for the contract */ - image?: FileBufferOrString; + image?: FileOrBufferOrString; /** * Optional url for the contract */ @@ -111,7 +111,7 @@ export interface MarketplaceContractDeployMetadata { /** * Optional image for the contract */ - image?: FileBufferOrString; + image?: FileOrBufferOrString; /** * Optional url for the contract */ @@ -146,7 +146,7 @@ export interface VoteContractDeployMetadata { /** * Optional image for the contract */ - image?: FileBufferOrString; + image?: FileOrBufferOrString; /** * Optional url for the contract */ @@ -209,7 +209,7 @@ export interface SplitContractDeployMetadata { /** * Optional image for the contract */ - image?: FileBufferOrString; + image?: FileOrBufferOrString; /** * Optional url for the contract */ @@ -240,7 +240,7 @@ export interface MultiwrapContractDeployMetadata { /** * Optional image for the contract */ - image?: FileBufferOrString; + image?: FileOrBufferOrString; /** * Optional url for the contract */ diff --git a/packages/sdk/test/edition.test.ts b/packages/sdk/test/edition.test.ts index b4527113687..ee2a31ce2ba 100644 --- a/packages/sdk/test/edition.test.ts +++ b/packages/sdk/test/edition.test.ts @@ -98,7 +98,7 @@ describe("Edition Contract", async () => { }); it("should mint with URI", async () => { - const uri = await storage.uploadMetadata({ + const uri = await storage.upload({ name: "Test1", }); const tx = await bundleContract.mint({ @@ -111,7 +111,7 @@ describe("Edition Contract", async () => { }); it("should mint batch with URI", async () => { - const uri = await storage.uploadMetadata({ + const uri = await storage.upload({ name: "Test1", }); await bundleContract.mintBatch([ diff --git a/packages/sdk/test/hooks.ts b/packages/sdk/test/hooks.ts index 0f674a90590..6745b14bc7f 100644 --- a/packages/sdk/test/hooks.ts +++ b/packages/sdk/test/hooks.ts @@ -42,7 +42,7 @@ import { TWRegistry__factory, VoteERC20__factory, } from "@thirdweb-dev/contracts-js"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { ethers } from "ethers"; import hardhat from "hardhat"; @@ -59,7 +59,7 @@ let sdk: ThirdwebSDK; const ipfsGatewayUrl = DEFAULT_IPFS_GATEWAY; let signer: SignerWithAddress; let signers: SignerWithAddress[]; -let storage: IStorage; +let storage: ThirdwebStorage; let implementations: { [key in ContractType]?: string }; const fastForwardTime = async (timeInSeconds: number): Promise => { diff --git a/packages/sdk/test/mock/MockStorage.ts b/packages/sdk/test/mock/MockStorage.ts index dcdd752a2d5..a9e69f792a1 100644 --- a/packages/sdk/test/mock/MockStorage.ts +++ b/packages/sdk/test/mock/MockStorage.ts @@ -1,15 +1,13 @@ import { NotFoundError } from "../../src"; import { JsonObject } from "../../src/core/types"; -import { - FileOrBuffer, - isBufferInstance, - isFileInstance, - IStorage, - UploadResult, -} from "@thirdweb-dev/storage"; +import { FileOrBuffer, ThirdwebStorage } from "@thirdweb-dev/storage"; import { v4 as uuidv4 } from "uuid"; -export class MockStorage implements IStorage { +export function MockStorage() { + return new ThirdwebStorage(); +} + +export class MocStorage implements IStorage { private objects: { [key: string]: any } = {}; private folders: { [cid: string]: { [id: string]: any } } = {}; diff --git a/packages/sdk/test/nft-drop.test.ts b/packages/sdk/test/nft-drop.test.ts index c08496c0f2a..4c1a4902cf2 100644 --- a/packages/sdk/test/nft-drop.test.ts +++ b/packages/sdk/test/nft-drop.test.ts @@ -42,7 +42,7 @@ describe("NFT Drop Contract", async () => { }); it("should lazy mint with URI", async () => { - const uri = await storage.uploadMetadata({ + const uri = await storage.upload({ name: "Test1", }); await dropContract.createBatch([uri]); diff --git a/packages/sdk/test/nft.test.ts b/packages/sdk/test/nft.test.ts index c543cb3efcf..54372fa12d3 100644 --- a/packages/sdk/test/nft.test.ts +++ b/packages/sdk/test/nft.test.ts @@ -102,7 +102,7 @@ describe("NFT Contract", async () => { }); it("should mint with URI", async () => { - const uri = await storage.uploadMetadata({ + const uri = await storage.upload({ name: "Test1", }); await nftContract.mint(uri); @@ -112,7 +112,7 @@ describe("NFT Contract", async () => { }); it("should mint batch with URI", async () => { - const uri = await storage.uploadMetadata({ + const uri = await storage.upload({ name: "Test1", }); await nftContract.mintBatch([uri]); diff --git a/packages/sdk/test/publisher.test.ts b/packages/sdk/test/publisher.test.ts index 0f57ec22e5b..ead91f5d83c 100644 --- a/packages/sdk/test/publisher.test.ts +++ b/packages/sdk/test/publisher.test.ts @@ -11,7 +11,7 @@ import { DropERC721__factory, TokenERC721__factory, } from "@thirdweb-dev/contracts-js"; -import { IpfsStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { expect } from "chai"; import { ethers } from "ethers"; import { readFileSync } from "fs"; @@ -21,7 +21,7 @@ global.fetch = require("cross-fetch"); export const uploadContractMetadata = async ( contractName: string, - storage: IpfsStorage, + storage: ThirdwebStorage, ) => { const buildinfo = JSON.parse( readFileSync("test/test_abis/hardhat-build-info.json", "utf-8"), @@ -29,14 +29,18 @@ export const uploadContractMetadata = async ( const info = buildinfo.output.contracts[`contracts/${contractName}.sol`][contractName]; const bytecode = `0x${info.evm.bytecode.object}`; - const metadataUri = await storage.uploadSingle(info.metadata); - const bytecodeUri = await storage.uploadSingle(bytecode); + const metadataUri = await storage.upload(info.metadata, { + uploadWithoutDirectory: true, + }); + const bytecodeUri = await storage.upload(bytecode, { + uploadWithoutDirectory: true, + }); const model = { name: contractName, metadataUri: `ipfs://${metadataUri}`, bytecodeUri: `ipfs://${bytecodeUri}`, }; - return await storage.uploadMetadata(model); + return await storage.upload(model); }; describe("Publishing", async () => { diff --git a/packages/sdk/test/signature-drop.test.ts b/packages/sdk/test/signature-drop.test.ts index 88613e11e27..b1489851558 100644 --- a/packages/sdk/test/signature-drop.test.ts +++ b/packages/sdk/test/signature-drop.test.ts @@ -180,7 +180,7 @@ describe("Signature drop tests", async () => { }); it("should mint with URI", async () => { - const uri = await storage.uploadMetadata({ + const uri = await storage.upload({ name: "Test1", }); const toSign = { @@ -194,10 +194,10 @@ describe("Signature drop tests", async () => { }); it("should mint batch with URI", async () => { - const uri1 = await storage.uploadMetadata({ + const uri1 = await storage.upload({ name: "Test1", }); - const uri2 = await storage.uploadMetadata({ + const uri2 = await storage.upload({ name: "Test2", }); const toSign1 = { diff --git a/packages/sdk/test/signature-mint-1155.test.ts b/packages/sdk/test/signature-mint-1155.test.ts index 9084faf7a32..8de86587936 100644 --- a/packages/sdk/test/signature-mint-1155.test.ts +++ b/packages/sdk/test/signature-mint-1155.test.ts @@ -169,7 +169,7 @@ describe("Edition sig minting", async () => { }); it("should mint with URI", async () => { - const uri = await storage.uploadMetadata({ + const uri = await storage.upload({ name: "Test1", }); const toSign = { @@ -183,10 +183,10 @@ describe("Edition sig minting", async () => { }); it("should mint batch with URI", async () => { - const uri1 = await storage.uploadMetadata({ + const uri1 = await storage.upload({ name: "Test1", }); - const uri2 = await storage.uploadMetadata({ + const uri2 = await storage.upload({ name: "Test2", }); const toSign1 = { diff --git a/packages/sdk/test/signature-mint-721.test.ts b/packages/sdk/test/signature-mint-721.test.ts index 96f507a36e2..07ab58d3482 100644 --- a/packages/sdk/test/signature-mint-721.test.ts +++ b/packages/sdk/test/signature-mint-721.test.ts @@ -188,7 +188,7 @@ describe("NFT sig minting", async () => { }); it("should mint with URI", async () => { - const uri = await storage.uploadMetadata({ + const uri = await storage.upload({ name: "Test1", }); const toSign = { @@ -201,10 +201,10 @@ describe("NFT sig minting", async () => { }); it("should mint batch with URI", async () => { - const uri1 = await storage.uploadMetadata({ + const uri1 = await storage.upload({ name: "Test1", }); - const uri2 = await storage.uploadMetadata({ + const uri2 = await storage.upload({ name: "Test2", }); const toSign1 = { diff --git a/packages/sdk/test/snapshot.test.ts b/packages/sdk/test/snapshot.test.ts index 06eef40c168..ecf050a73b4 100644 --- a/packages/sdk/test/snapshot.test.ts +++ b/packages/sdk/test/snapshot.test.ts @@ -1,6 +1,6 @@ import { createSnapshot, Snapshot } from "../src/index"; import { MockStorage } from "./mock/MockStorage"; -import { IStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; const chai = require("chai"); const deepEqualInAnyOrder = require("deep-equal-in-any-order"); @@ -28,7 +28,7 @@ describe("Snapshots", async () => { maxClaimable: 0, })); - let storage: IStorage; + let storage: ThirdwebStorage; beforeEach(async () => { storage = new MockStorage(); diff --git a/packages/storage/src/core/storage.ts b/packages/storage/src/core/storage.ts index 19b0ab327a6..79d0445cb8c 100644 --- a/packages/storage/src/core/storage.ts +++ b/packages/storage/src/core/storage.ts @@ -10,7 +10,7 @@ import { IpfsUploadBatchOptions, IStorageDownloader, IStorageUploader, - JsonObject, + Json, UploadDataSchema, UploadOptions, } from "../types"; @@ -44,19 +44,19 @@ export class ThirdwebStorage { ) as TJSON; } - async upload(data: JsonObject | FileOrBuffer, options?: T): Promise { + async upload(data: Json | FileOrBuffer, options?: T): Promise { const [uri] = await this.uploadBatch( - [data] as JsonObject[] | FileOrBuffer[], + [data] as Json[] | FileOrBuffer[], options, ); return uri; } async uploadBatch( - data: JsonObject[] | FileOrBuffer[], + data: Json[] | FileOrBuffer[], options?: T, ): Promise { - const parsed: JsonObject[] | FileOrBuffer[] = UploadDataSchema.parse(data); + const parsed: Json[] | FileOrBuffer[] = UploadDataSchema.parse(data); const { success: isFileArray } = FileOrBufferSchema.safeParse(parsed[0]); // If data is an array of files, pass it through to upload directly @@ -66,19 +66,16 @@ export class ThirdwebStorage { // Otherwise it is an array of JSON objects, so we have to prepare it first const metadata = ( - await this.uploadAndReplaceFilesWithHashes( - parsed as JsonObject[], - options, - ) + await this.uploadAndReplaceFilesWithHashes(parsed as Json[], options) ).map((item) => JSON.stringify(item)); return this.uploader.uploadBatch(metadata, options); } private async uploadAndReplaceFilesWithHashes( - data: JsonObject[], + data: Json[], options?: T, - ): Promise { + ): Promise { let cleaned = data; // TODO: Gateway URLs should probably be top-level since both uploader and downloader need them if (this.uploader.gatewayUrls || this.downloader.gatewayUrls) { @@ -86,7 +83,7 @@ export class ThirdwebStorage { cleaned = replaceObjectGatewayUrlsWithSchemes( cleaned, this.uploader.gatewayUrls || this.downloader.gatewayUrls, - ) as JsonObject[]; + ) as Json[]; if (options?.uploadWithGatewayUrl || this.uploader.uploadWithGatewayUrl) { // If flag is set, replace all schemes with their preferred gateway URL @@ -94,7 +91,7 @@ export class ThirdwebStorage { cleaned = replaceObjectSchemesWithGatewayUrls( cleaned, this.uploader.gatewayUrls || this.downloader.gatewayUrls, - ) as JsonObject[]; + ) as Json[]; } } @@ -109,6 +106,6 @@ export class ThirdwebStorage { const uris = await this.uploader.uploadBatch(files); // Recurse through data and replace files with hashes - return replaceObjectFilesWithUris(cleaned, uris) as JsonObject[]; + return replaceObjectFilesWithUris(cleaned, uris) as Json[]; } } diff --git a/packages/storage/src/types/data.ts b/packages/storage/src/types/data.ts index 175acffde89..823192ebf4f 100644 --- a/packages/storage/src/types/data.ts +++ b/packages/storage/src/types/data.ts @@ -56,7 +56,7 @@ export type JsonObject = { [key: string]: Json }; export const UploadDataSchema = z.union( [ - z.array(JsonObjectSchema).nonempty({ + z.array(JsonSchema).nonempty({ message: "Cannot pass an empty array.", }), z.array(FileOrBufferSchema).nonempty({ From aa0b1887e3f0333dfd4e575f41f4d91bec48601e Mon Sep 17 00:00:00 2001 From: adam-maj Date: Fri, 16 Sep 2022 16:46:19 -0700 Subject: [PATCH 12/19] Update storage test cases --- packages/sdk/src/common/nft.ts | 2 +- packages/sdk/test/hooks.ts | 2 +- packages/sdk/test/mock/MockStorage.ts | 184 +++++++------------------- packages/sdk/test/snapshot.test.ts | 4 +- packages/storage/src/common/index.ts | 2 + packages/storage/src/index.ts | 1 + 6 files changed, 53 insertions(+), 142 deletions(-) create mode 100644 packages/storage/src/common/index.ts diff --git a/packages/sdk/src/common/nft.ts b/packages/sdk/src/common/nft.ts index 373494dcd54..c52d4810fb8 100644 --- a/packages/sdk/src/common/nft.ts +++ b/packages/sdk/src/common/nft.ts @@ -55,7 +55,7 @@ export async function fetchTokenMetadata( ); try { jsonMetadata = await storage.downloadJSON(unparsedTokenIdUri); - } catch (e) { + } catch (e: any) { console.warn( `failed to get token metadata: ${JSON.stringify({ tokenId: tokenId.toString(), diff --git a/packages/sdk/test/hooks.ts b/packages/sdk/test/hooks.ts index 6745b14bc7f..8a2e65cf540 100644 --- a/packages/sdk/test/hooks.ts +++ b/packages/sdk/test/hooks.ts @@ -221,7 +221,7 @@ export const mochaHooks = { // eslint-disable-next-line turbo/no-undeclared-env-vars process.env.contractPublisherAddress = contractPublisher.address; - storage = new MockStorage(); + storage = MockStorage(); sdk = new ThirdwebSDK( signer, { diff --git a/packages/sdk/test/mock/MockStorage.ts b/packages/sdk/test/mock/MockStorage.ts index a9e69f792a1..8445c817742 100644 --- a/packages/sdk/test/mock/MockStorage.ts +++ b/packages/sdk/test/mock/MockStorage.ts @@ -1,49 +1,31 @@ -import { NotFoundError } from "../../src"; -import { JsonObject } from "../../src/core/types"; -import { FileOrBuffer, ThirdwebStorage } from "@thirdweb-dev/storage"; +import { + DEFAULT_GATEWAY_URLS, + FileOrBufferOrString, + GatewayUrls, + IpfsUploadBatchOptions, + IpfsUploaderOptions, + isBufferInstance, + isFileInstance, + IStorageDownloader, + IStorageUploader, + ThirdwebStorage, +} from "@thirdweb-dev/storage"; import { v4 as uuidv4 } from "uuid"; -export function MockStorage() { - return new ThirdwebStorage(); -} - -export class MocStorage implements IStorage { - private objects: { [key: string]: any } = {}; - private folders: { [cid: string]: { [id: string]: any } } = {}; - - public async upload( - data: string | FileOrBuffer, - _contractAddress?: string, - _signerAddress?: string, - ): Promise { - const uuid = uuidv4(); - let serializedData = ""; - - if (isFileInstance(data)) { - serializedData = await data.text(); - } else if (isBufferInstance(data)) { - serializedData = data.toString(); - } else if (typeof data === "string") { - serializedData = data; - } +// Store mapping of URIs to files/objects +const ipfs: Record = {}; - const key = `mock://${uuid}`; - this.objects[uuid] = serializedData; - return Promise.resolve(key); - } - - public async uploadBatch( - files: (string | FileOrBuffer)[], - fileStartNumber?: number, - _contractAddress?: string, - _signerAddress?: string, - ): Promise { +class MockStorageUploader implements IStorageUploader { + async uploadBatch( + data: FileOrBufferOrString[], + options?: IpfsUploadBatchOptions | undefined, + ): Promise { const cid = uuidv4(); const uris: string[] = []; - this.folders[cid] = {}; + ipfs[cid] = {}; - let index = fileStartNumber ? fileStartNumber : 0; - for (const file of files) { + let index = options?.rewriteFileNames?.fileStartNumber || 0; + for (const file of data) { let contents: string; if (isFileInstance(file)) { contents = await file.text(); @@ -56,113 +38,39 @@ export class MocStorage implements IStorage { ? file.data.toString() : file.data; const name = file.name ? file.name : `file_${index}`; - this.folders.cid[name] = contents; + ipfs[cid][name] = contents; + uris.push(`mock://${cid}/${name}`); continue; } - this.folders[cid][index.toString()] = contents; - uris.push(`${cid}/${index}`); - index += 1; - } - return Promise.resolve({ - baseUri: `mock://${cid}`, - uris, - }); - } - - public async getUploadToken(_contractAddress: string): Promise { - return Promise.resolve("mock-token"); - } - - private _get(hash: string): string { - hash = hash.replace("mock://", "").replace("fake://", ""); - const split = hash.split("/"); - if (split.length === 1) { - if (hash in this.objects) { - return this.objects[hash]; - } else { - throw new NotFoundError(hash); - } - } - const [cid, index] = split; - if (!(cid in this.folders)) { - throw new NotFoundError(cid); - } - if (!(index in this.folders[cid])) { - throw new NotFoundError(`${cid}/${index}`); + ipfs[cid][index.toString()] = contents; + uris.push(`mock://${cid}/${index}`); + index += 1; } - return this.folders[cid][index.toString()]; - } - get(hash: string): Promise> { - return Promise.resolve(JSON.parse(this._get(hash))); + return uris; } +} - getRaw(hash: string): Promise { - return Promise.resolve(this._get(hash)); - } - - public async uploadMetadata( - metadata: JsonObject, - contractAddress?: string, - _signerAddress?: string, - ): Promise { - // since there's only single object, always use the first index - const { uris } = await this.uploadMetadataBatch( - [metadata], - 0, - contractAddress, - ); - - return uris[0]; - } - - public async uploadMetadataBatch( - metadatas: JsonObject[], - fileStartNumber?: number, - contractAddress?: string, - signerAddress?: string, - ): Promise { - await this.batchUploadProperties(metadatas); - - const metadataToUpload: string[] = metadatas.map((m: any) => - JSON.stringify(m), - ); +class MockStorageDownloader implements IStorageDownloader { + gatewayUrls: GatewayUrls = DEFAULT_GATEWAY_URLS; - const { baseUri: cid } = await this.uploadBatch( - metadataToUpload, - fileStartNumber, - contractAddress, - signerAddress, - ); - const baseUri = `${cid}/`; + async download(url: string): Promise { + const [cid, name] = url.replace("mock://", "").split("/"); + const data = ipfs[cid][name]; return { - uris: metadataToUpload.map((_, i) => `${baseUri}${i}`), - baseUri, - }; - } - - private async uploadProperties(object: Record): Promise { - const keys = Object.keys(object).sort(); - for (const key in keys) { - const val = object[keys[key]]; - const shouldUpload = isFileInstance(val) || isBufferInstance(val); - if (shouldUpload) { - object[keys[key]] = await this.upload(val); - } - - if (typeof val === "object") { - await this.uploadProperties(val); - } - } + async json() { + return Promise.resolve(JSON.parse(data)); + }, + async text() { + return Promise.resolve(data); + }, + } as Response; } +} - private async batchUploadProperties(metadatas: JsonObject[]): Promise { - for (const file of metadatas) { - if (typeof file === "string") { - continue; - } - await this.uploadProperties(file); - } - } +export function MockStorage(): ThirdwebStorage { + const uploader = new MockStorageUploader(); + const downloader = new MockStorageDownloader(); + return new ThirdwebStorage(uploader, downloader); } diff --git a/packages/sdk/test/snapshot.test.ts b/packages/sdk/test/snapshot.test.ts index ecf050a73b4..c2f786d2644 100644 --- a/packages/sdk/test/snapshot.test.ts +++ b/packages/sdk/test/snapshot.test.ts @@ -31,7 +31,7 @@ describe("Snapshots", async () => { let storage: ThirdwebStorage; beforeEach(async () => { - storage = new MockStorage(); + storage = MockStorage(); }); beforeEach(async () => { @@ -88,7 +88,7 @@ describe("Snapshots", async () => { }); it("should upload the snapshot to storage", async () => { - const rawSnapshotJson = await storage.get(uri); + const rawSnapshotJson = await storage.download(uri); expect(rawSnapshotJson).to.deep.equalInAnyOrder(snapshot); }); }); diff --git a/packages/storage/src/common/index.ts b/packages/storage/src/common/index.ts new file mode 100644 index 00000000000..849a6e94743 --- /dev/null +++ b/packages/storage/src/common/index.ts @@ -0,0 +1,2 @@ +export * from "./urls"; +export * from "./utils"; diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index d778e12df16..b6d8929b937 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -1,2 +1,3 @@ export * from "./core"; export * from "./types"; +export * from "./common"; From ee4d9d39fe56cc8e94bf0c0b54bf8c5f7a7f6d0f Mon Sep 17 00:00:00 2001 From: adam-maj Date: Fri, 16 Sep 2022 16:50:42 -0700 Subject: [PATCH 13/19] Update all packages using storage --- packages/cli/src/common/ci-installer.ts | 6 +++--- packages/cli/src/common/processor.ts | 19 ++++++++++++------- packages/react/src/Provider.tsx | 4 ++-- packages/react/src/hooks/useReadonlySDK.ts | 4 ++-- packages/sdk/test/mock/MockStorage.ts | 1 - 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/packages/cli/src/common/ci-installer.ts b/packages/cli/src/common/ci-installer.ts index 0313de166ed..a8c10e3a724 100644 --- a/packages/cli/src/common/ci-installer.ts +++ b/packages/cli/src/common/ci-installer.ts @@ -1,5 +1,5 @@ import { spinner } from "../core/helpers/logger"; -import { IpfsStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import { existsSync, mkdirSync, writeFileSync } from "fs"; import path, { join } from "path"; @@ -13,10 +13,10 @@ export async function installGithubAction(options: any) { : path.resolve(`${projectPath}/${options.path}`); projectPath = resolvedPath; } - const storage = new IpfsStorage(); + const storage = new ThirdwebStorage(); const log = spinner("Installing thirdweb Github Action..."); try { - const ghActionData = await storage.getRaw(ghActionHash); + const ghActionData = await (await storage.download(ghActionHash)).text(); const dir = join(projectPath, ".github/workflows"); if (!existsSync(dir)) { mkdirSync(dir, { recursive: true }); diff --git a/packages/cli/src/common/processor.ts b/packages/cli/src/common/processor.ts index 168f308d0a4..22b00c580a2 100644 --- a/packages/cli/src/common/processor.ts +++ b/packages/cli/src/common/processor.ts @@ -5,7 +5,7 @@ import { execute } from "../core/helpers/exec"; import { error, info, logger, spinner } from "../core/helpers/logger"; import { createContractsPrompt } from "../core/helpers/selector"; import { ContractPayload } from "../core/interfaces/ContractPayload"; -import { IpfsStorage } from "@thirdweb-dev/storage"; +import { ThirdwebStorage } from "@thirdweb-dev/storage"; import chalk from "chalk"; import { readFileSync } from "fs"; import path from "path"; @@ -15,7 +15,7 @@ export async function processProject( command: "deploy" | "release", ) { // TODO: allow overriding the default storage - const storage = new IpfsStorage(); + const storage = new ThirdwebStorage(); logger.setSettings({ minLevel: options.debug ? "debug" : "info", @@ -129,7 +129,9 @@ export async function processProject( if (file.includes(soliditySDKPackage)) { usesSoliditySDK = true; } - return await storage.uploadSingle(file); + return await storage.upload(file, { + uploadWithoutDirectory: true, + }); }), ); } @@ -140,14 +142,16 @@ export async function processProject( const metadataURIs = await Promise.all( selectedContracts.map(async (c) => { logger.debug(`Uploading ${c.name}...`); - const hash = await storage.uploadSingle(c.metadata); + const hash = await storage.upload(c.metadata, { + uploadWithoutDirectory: true, + }); return `ipfs://${hash}`; }), ); // Upload batch all bytecodes const bytecodes = selectedContracts.map((c) => c.bytecode); - const { uris: bytecodeURIs } = await storage.uploadBatch(bytecodes); + const bytecodeURIs = await storage.uploadBatch(bytecodes); const combinedContents = selectedContracts.map((c, i) => { // attach analytics blob to metadata @@ -169,13 +173,14 @@ export async function processProject( let combinedURIs: string[] = []; if (combinedContents.length === 1) { // use upload single if only one contract to get a clean IPFS hash - const metadataUri = await storage.uploadSingle( + const metadataUri = await storage.upload( JSON.stringify(combinedContents[0]), + { uploadWithoutDirectory: true }, ); combinedURIs.push(metadataUri); } else { // otherwise upload batch - const { uris } = await storage.uploadMetadataBatch(combinedContents); + const uris = await storage.uploadBatch(combinedContents); combinedURIs = uris; } diff --git a/packages/react/src/Provider.tsx b/packages/react/src/Provider.tsx index 97e869d30ff..9edfa2d6477 100644 --- a/packages/react/src/Provider.tsx +++ b/packages/react/src/Provider.tsx @@ -27,7 +27,7 @@ import { getProviderForNetwork, SDKOptionsOutput, } from "@thirdweb-dev/sdk"; -import type { IStorage } from "@thirdweb-dev/storage"; +import type { ThirdwebStorage } from "@thirdweb-dev/storage"; import { Signer } from "ethers"; import React, { createContext, useEffect, useMemo } from "react"; import invariant from "tiny-invariant"; @@ -181,7 +181,7 @@ export interface ThirdwebProviderProps< /** * The storage interface to use with the sdk. */ - storageInterface?: IStorage; + storageInterface?: ThirdwebStorage; /** * The react-query client to use. (Defaults to a default client.) diff --git a/packages/react/src/hooks/useReadonlySDK.ts b/packages/react/src/hooks/useReadonlySDK.ts index 4bee7e09dfa..dca4367b0d8 100644 --- a/packages/react/src/hooks/useReadonlySDK.ts +++ b/packages/react/src/hooks/useReadonlySDK.ts @@ -1,5 +1,5 @@ import { SDKOptions, ThirdwebSDK } from "@thirdweb-dev/sdk"; -import type { IStorage } from "@thirdweb-dev/storage"; +import type { ThirdwebStorage } from "@thirdweb-dev/storage"; import { useMemo } from "react"; /** @@ -8,7 +8,7 @@ import { useMemo } from "react"; export function useReadonlySDK( readonlyRpcUrl: string, sdkOptions: SDKOptions, - storageInterface?: IStorage, + storageInterface?: ThirdwebStorage, ): ThirdwebSDK { return useMemo(() => { return new ThirdwebSDK( diff --git a/packages/sdk/test/mock/MockStorage.ts b/packages/sdk/test/mock/MockStorage.ts index 8445c817742..d4ef6bd5fe2 100644 --- a/packages/sdk/test/mock/MockStorage.ts +++ b/packages/sdk/test/mock/MockStorage.ts @@ -3,7 +3,6 @@ import { FileOrBufferOrString, GatewayUrls, IpfsUploadBatchOptions, - IpfsUploaderOptions, isBufferInstance, isFileInstance, IStorageDownloader, From 002cbefc17e368644f781e738893fe27c231f95d Mon Sep 17 00:00:00 2001 From: adam-maj Date: Fri, 16 Sep 2022 17:02:30 -0700 Subject: [PATCH 14/19] Update mock storage for Solana --- packages/solana/src/classes/deployer.ts | 4 +- .../solana/src/types/contracts/nft-drop.ts | 2 + packages/solana/test/mock/MockStorage.ts | 182 ++++-------------- packages/solana/test/nft-drop.test.ts | 2 +- 4 files changed, 46 insertions(+), 144 deletions(-) diff --git a/packages/solana/src/classes/deployer.ts b/packages/solana/src/classes/deployer.ts index fa5db6ae8d3..8d5ffe1ede3 100644 --- a/packages/solana/src/classes/deployer.ts +++ b/packages/solana/src/classes/deployer.ts @@ -6,7 +6,7 @@ import { } from "../types/contracts"; import { NFTDropConditionsOutputSchema, - NFTDropMetadataInput, + NFTDropContractInput, } from "../types/contracts/nft-drop"; import { enforceCreator } from "./helpers/creators-helper"; import { @@ -99,7 +99,7 @@ export class Deployer { return collectionNft.mint.address.toBase58(); } - async createNftDrop(metadata: NFTDropMetadataInput): Promise { + async createNftDrop(metadata: NFTDropContractInput): Promise { const collectionInfo = NFTCollectionMetadataInputSchema.parse(metadata); const candyMachineInfo = NFTDropConditionsOutputSchema.parse(metadata); const uri = await this.storage.upload(collectionInfo); diff --git a/packages/solana/src/types/contracts/nft-drop.ts b/packages/solana/src/types/contracts/nft-drop.ts index 70111fbbea9..a3d01cb2e0e 100644 --- a/packages/solana/src/types/contracts/nft-drop.ts +++ b/packages/solana/src/types/contracts/nft-drop.ts @@ -48,4 +48,6 @@ export const NFTDropConditionsOutputSchema = z.object({ export const NFTDropContractInputSchema = NFTCollectionMetadataInputSchema.merge(NFTDropConditionsInputSchema); +export type NFTDropContractInput = z.input; + export type NFTDropMetadataInput = z.input; diff --git a/packages/solana/test/mock/MockStorage.ts b/packages/solana/test/mock/MockStorage.ts index d95c2a4c45a..d4ef6bd5fe2 100644 --- a/packages/solana/test/mock/MockStorage.ts +++ b/packages/solana/test/mock/MockStorage.ts @@ -1,56 +1,30 @@ import { - FileOrBuffer, + DEFAULT_GATEWAY_URLS, + FileOrBufferOrString, + GatewayUrls, + IpfsUploadBatchOptions, isBufferInstance, isFileInstance, - IStorage, - JsonObject, - UploadResult, + IStorageDownloader, + IStorageUploader, + ThirdwebStorage, } from "@thirdweb-dev/storage"; import { v4 as uuidv4 } from "uuid"; -class NotFoundError extends Error { - constructor(message: string) { - super(message); - } -} - -export class MockStorage implements IStorage { - private objects: { [key: string]: any } = {}; - private folders: { [cid: string]: { [id: string]: any } } = {}; - - public async upload( - data: string | FileOrBuffer, - _contractAddress?: string, - _signerAddress?: string, - ): Promise { - const uuid = uuidv4(); - let serializedData = ""; - - if (isFileInstance(data)) { - serializedData = await data.text(); - } else if (isBufferInstance(data)) { - serializedData = data.toString(); - } else if (typeof data === "string") { - serializedData = data; - } - - const key = `mock://${uuid}`; - this.objects[uuid] = serializedData; - return Promise.resolve(key); - } +// Store mapping of URIs to files/objects +const ipfs: Record = {}; - public async uploadBatch( - files: (string | FileOrBuffer)[], - fileStartNumber?: number, - _contractAddress?: string, - _signerAddress?: string, - ): Promise { +class MockStorageUploader implements IStorageUploader { + async uploadBatch( + data: FileOrBufferOrString[], + options?: IpfsUploadBatchOptions | undefined, + ): Promise { const cid = uuidv4(); const uris: string[] = []; - this.folders[cid] = {}; + ipfs[cid] = {}; - let index = fileStartNumber ? fileStartNumber : 0; - for (const file of files) { + let index = options?.rewriteFileNames?.fileStartNumber || 0; + for (const file of data) { let contents: string; if (isFileInstance(file)) { contents = await file.text(); @@ -63,113 +37,39 @@ export class MockStorage implements IStorage { ? file.data.toString() : file.data; const name = file.name ? file.name : `file_${index}`; - this.folders.cid[name] = contents; + ipfs[cid][name] = contents; + uris.push(`mock://${cid}/${name}`); continue; } - this.folders[cid][index.toString()] = contents; - uris.push(`${cid}/${index}`); - index += 1; - } - return Promise.resolve({ - baseUri: `mock://${cid}`, - uris, - }); - } - - public async getUploadToken(_contractAddress: string): Promise { - return Promise.resolve("mock-token"); - } - - private _get(hash: string): string { - hash = hash.replace("mock://", "").replace("fake://", ""); - const split = hash.split("/"); - if (split.length === 1) { - if (hash in this.objects) { - return this.objects[hash]; - } else { - throw new NotFoundError(hash); - } - } - const [cid, index] = split; - if (!(cid in this.folders)) { - throw new NotFoundError(cid); - } - if (!(index in this.folders[cid])) { - throw new NotFoundError(`${cid}/${index}`); + ipfs[cid][index.toString()] = contents; + uris.push(`mock://${cid}/${index}`); + index += 1; } - return this.folders[cid][index.toString()]; - } - - get(hash: string): Promise> { - return Promise.resolve(JSON.parse(this._get(hash))); - } - - getRaw(hash: string): Promise { - return Promise.resolve(this._get(hash)); - } - public async uploadMetadata( - metadata: JsonObject, - contractAddress?: string, - _signerAddress?: string, - ): Promise { - // since there's only single object, always use the first index - const { uris } = await this.uploadMetadataBatch( - [metadata], - 0, - contractAddress, - ); - - return uris[0]; + return uris; } +} - public async uploadMetadataBatch( - metadatas: JsonObject[], - fileStartNumber?: number, - contractAddress?: string, - signerAddress?: string, - ): Promise { - await this.batchUploadProperties(metadatas); - - const metadataToUpload: string[] = metadatas.map((m: any) => - JSON.stringify(m), - ); +class MockStorageDownloader implements IStorageDownloader { + gatewayUrls: GatewayUrls = DEFAULT_GATEWAY_URLS; - const { baseUri: cid } = await this.uploadBatch( - metadataToUpload, - fileStartNumber, - contractAddress, - signerAddress, - ); - const baseUri = `${cid}/`; + async download(url: string): Promise { + const [cid, name] = url.replace("mock://", "").split("/"); + const data = ipfs[cid][name]; return { - uris: metadataToUpload.map((_, i) => `${baseUri}${i}`), - baseUri, - }; - } - - private async uploadProperties(object: Record): Promise { - const keys = Object.keys(object).sort(); - for (const key in keys) { - const val = object[keys[key]]; - const shouldUpload = isFileInstance(val) || isBufferInstance(val); - if (shouldUpload) { - object[keys[key]] = await this.upload(val); - } - - if (typeof val === "object") { - await this.uploadProperties(val); - } - } + async json() { + return Promise.resolve(JSON.parse(data)); + }, + async text() { + return Promise.resolve(data); + }, + } as Response; } +} - private async batchUploadProperties(metadatas: JsonObject[]): Promise { - for (const file of metadatas) { - if (typeof file === "string") { - continue; - } - await this.uploadProperties(file); - } - } +export function MockStorage(): ThirdwebStorage { + const uploader = new MockStorageUploader(); + const downloader = new MockStorageDownloader(); + return new ThirdwebStorage(uploader, downloader); } diff --git a/packages/solana/test/nft-drop.test.ts b/packages/solana/test/nft-drop.test.ts index 96642a411a8..d7291d01944 100644 --- a/packages/solana/test/nft-drop.test.ts +++ b/packages/solana/test/nft-drop.test.ts @@ -8,7 +8,7 @@ describe("NFTDrop", async () => { before(async () => { const address = await sdk.deployer.createNftDrop({ - name: "Test Drop", + name: "NFT Drop #1", price: 0, sellerFeeBasisPoints: 0, itemsAvailable: 5, From 48571cc56db49e60631ba5c595a1a36078b61b8c Mon Sep 17 00:00:00 2001 From: adam-maj Date: Fri, 16 Sep 2022 17:02:51 -0700 Subject: [PATCH 15/19] Update mock storage usage --- packages/solana/test/before-setup.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/solana/test/before-setup.ts b/packages/solana/test/before-setup.ts index 66f47ae9354..c5f488c0535 100644 --- a/packages/solana/test/before-setup.ts +++ b/packages/solana/test/before-setup.ts @@ -12,7 +12,7 @@ export const createTestSDK = async ( solsToAirdrop: number = 100, ): Promise => { const connection = new Connection("http://localhost:8899", "confirmed"); - const sdk = new ThirdwebSDK(connection, new MockStorage()); + const sdk = new ThirdwebSDK(connection, MockStorage()); const wallet = Keypair.generate(); const amman = Amman.instance({ knownLabels: { From 403e8d0006e829cb8f68838566597a2972853ac9 Mon Sep 17 00:00:00 2001 From: adam-maj Date: Fri, 16 Sep 2022 18:00:53 -0700 Subject: [PATCH 16/19] Dont parse schema on uploadBatch --- packages/sdk/test/edition.test.ts | 3 --- packages/sdk/test/snapshot.test.ts | 2 +- packages/storage/src/core/storage.ts | 12 +++++++----- packages/storage/src/types/data.ts | 17 ----------------- 4 files changed, 8 insertions(+), 26 deletions(-) diff --git a/packages/sdk/test/edition.test.ts b/packages/sdk/test/edition.test.ts index ee2a31ce2ba..e1602ba71d9 100644 --- a/packages/sdk/test/edition.test.ts +++ b/packages/sdk/test/edition.test.ts @@ -241,14 +241,12 @@ describe("Edition Contract", async () => { { metadata: { name: "Test0", - image: "ipfs://myownipfs0", }, supply: 5, }, { metadata: { name: "Test1", - image: "ipfs://myownipfs1", }, supply: 5, }, @@ -258,7 +256,6 @@ describe("Edition Contract", async () => { let i = 0; nfts.forEach((nft) => { expect(nft.metadata.name).to.be.equal(`Test${i}`); - expect(nft.metadata.image).to.be.equal(`ipfs://myownipfs${i}`); i++; }); }); diff --git a/packages/sdk/test/snapshot.test.ts b/packages/sdk/test/snapshot.test.ts index c2f786d2644..bfe0e17ea60 100644 --- a/packages/sdk/test/snapshot.test.ts +++ b/packages/sdk/test/snapshot.test.ts @@ -88,7 +88,7 @@ describe("Snapshots", async () => { }); it("should upload the snapshot to storage", async () => { - const rawSnapshotJson = await storage.download(uri); + const rawSnapshotJson = await storage.downloadJSON(uri); expect(rawSnapshotJson).to.deep.equalInAnyOrder(snapshot); }); }); diff --git a/packages/storage/src/core/storage.ts b/packages/storage/src/core/storage.ts index 79d0445cb8c..390fa1d1559 100644 --- a/packages/storage/src/core/storage.ts +++ b/packages/storage/src/core/storage.ts @@ -11,7 +11,6 @@ import { IStorageDownloader, IStorageUploader, Json, - UploadDataSchema, UploadOptions, } from "../types"; import { StorageDownloader } from "./downloaders/storage-downloader"; @@ -56,17 +55,20 @@ export class ThirdwebStorage { data: Json[] | FileOrBuffer[], options?: T, ): Promise { - const parsed: Json[] | FileOrBuffer[] = UploadDataSchema.parse(data); - const { success: isFileArray } = FileOrBufferSchema.safeParse(parsed[0]); + if (!data.length) { + return []; + } + + const { success: isFileArray } = FileOrBufferSchema.safeParse(data[0]); // If data is an array of files, pass it through to upload directly if (isFileArray) { - return this.uploader.uploadBatch(parsed as FileOrBuffer[], options); + return this.uploader.uploadBatch(data as FileOrBuffer[], options); } // Otherwise it is an array of JSON objects, so we have to prepare it first const metadata = ( - await this.uploadAndReplaceFilesWithHashes(parsed as Json[], options) + await this.uploadAndReplaceFilesWithHashes(data as Json[], options) ).map((item) => JSON.stringify(item)); return this.uploader.uploadBatch(metadata, options); diff --git a/packages/storage/src/types/data.ts b/packages/storage/src/types/data.ts index 823192ebf4f..8e7ee17b275 100644 --- a/packages/storage/src/types/data.ts +++ b/packages/storage/src/types/data.ts @@ -53,20 +53,3 @@ export const JsonObjectSchema = z.record(z.string(), JsonSchema); export type Json = JsonLiteral | FileOrBuffer | JsonObject | Json[]; export type JsonObject = { [key: string]: Json }; - -export const UploadDataSchema = z.union( - [ - z.array(JsonSchema).nonempty({ - message: "Cannot pass an empty array.", - }), - z.array(FileOrBufferSchema).nonempty({ - message: "Cannot pass an empty array.", - }), - ], - { - invalid_type_error: - "Must pass a an array of all files or buffer objects or an array of all JSON objects.", - }, -); - -export type CleanedUploadData = string | FileOrBuffer; From 3b5e0b5af8fcfb55951875100ea0124947218dd3 Mon Sep 17 00:00:00 2001 From: adam-maj Date: Sun, 18 Sep 2022 18:30:32 -0700 Subject: [PATCH 17/19] Update file types to support browser --- packages/storage/src/types/data.ts | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/packages/storage/src/types/data.ts b/packages/storage/src/types/data.ts index 8e7ee17b275..cb9442e0269 100644 --- a/packages/storage/src/types/data.ts +++ b/packages/storage/src/types/data.ts @@ -1,9 +1,5 @@ import { z } from "zod"; -function isBrowser() { - return typeof window !== "undefined"; -} - const JsonLiteralSchema = z.union([ z.string(), z.number(), @@ -13,18 +9,24 @@ const JsonLiteralSchema = z.union([ type JsonLiteral = z.infer; +const isBrowser = () => typeof window !== "undefined"; const FileOrBufferUnionSchema = isBrowser() ? (z.instanceof(File) as z.ZodType>) - : (z.instanceof(Buffer) as z.ZodTypeAny); // TODO: fix, this is a hack to make browser happy for now + : (z.instanceof(Buffer) as z.ZodTypeAny); // @fixme, this is a hack to make browser happy for now export const FileOrBufferSchema = z.union([ FileOrBufferUnionSchema, z.object({ - data: z.union([z.instanceof(Buffer), z.string()]), + data: FileOrBufferUnionSchema, name: z.string(), }), ]); +export const FileOrBufferOrStringSchema = z.union([ + FileOrBufferSchema, + z.string(), +]); + export type FileOrBuffer = File | Buffer | BufferOrStringWithName; export type BufferOrStringWithName = { @@ -32,11 +34,6 @@ export type BufferOrStringWithName = { name: string; }; -export const FileOrBufferOrStringSchema = z.union([ - FileOrBufferSchema, - z.string(), -]); - export type FileOrBufferOrString = FileOrBuffer | string; const JsonSchema: z.ZodType = z.lazy(() => From caabed8c58222d19880a7b7db1bbb9e2e736790d Mon Sep 17 00:00:00 2001 From: adam-maj Date: Sun, 18 Sep 2022 18:44:05 -0700 Subject: [PATCH 18/19] Update json and file type imports from storage SDK --- packages/sdk/src/core/types.ts | 11 ----------- packages/sdk/src/schema/contracts/common/index.ts | 4 ++-- packages/sdk/src/schema/contracts/custom.ts | 4 ++-- packages/sdk/src/schema/shared.ts | 13 +------------ packages/sdk/src/schema/tokens/common/index.ts | 4 ++-- .../sdk/src/schema/tokens/common/properties.ts | 2 +- packages/solana/src/types/common.ts | 15 +-------------- packages/solana/src/types/contracts/index.ts | 4 ++-- packages/solana/src/types/nft.ts | 4 ++-- packages/storage/src/types/data.ts | 2 +- 10 files changed, 14 insertions(+), 49 deletions(-) diff --git a/packages/sdk/src/core/types.ts b/packages/sdk/src/core/types.ts index c7b40e98196..e7ab36f3e80 100644 --- a/packages/sdk/src/core/types.ts +++ b/packages/sdk/src/core/types.ts @@ -1,6 +1,5 @@ import type { CONTRACTS_MAP, PREBUILT_CONTRACTS_MAP } from "../contracts"; import type { SmartContract } from "../contracts/smart-contract"; -import { FileOrBuffer } from "@thirdweb-dev/storage"; import { BigNumber, BytesLike, CallOverrides, Signer, providers } from "ethers"; // --- utility types extracted from from ts-toolbelt --- // @@ -47,16 +46,6 @@ export type ValueOf = T[keyof T]; export type SignerOrProvider = Signer | providers.Provider; -export type BufferOrStringWithName = { - data: Buffer | string; - name?: string; -}; - -type JsonLiteral = boolean | null | number | string; -type JsonLiteralOrFileOrBuffer = JsonLiteral | FileOrBuffer; -export type Json = JsonLiteralOrFileOrBuffer | JsonObject | Json[]; -export type JsonObject = { [key: string]: Json }; - type TransactionResultWithMetadata = { receipt: providers.TransactionReceipt; data: () => Promise; diff --git a/packages/sdk/src/schema/contracts/common/index.ts b/packages/sdk/src/schema/contracts/common/index.ts index c47f654353e..f9a34684f79 100644 --- a/packages/sdk/src/schema/contracts/common/index.ts +++ b/packages/sdk/src/schema/contracts/common/index.ts @@ -1,5 +1,5 @@ -import { AddressSchema, BasisPointsSchema, JsonSchema } from "../../shared"; -import { FileOrBufferOrStringSchema } from "@thirdweb-dev/storage"; +import { AddressSchema, BasisPointsSchema } from "../../shared"; +import { FileOrBufferOrStringSchema, JsonSchema } from "@thirdweb-dev/storage"; import { constants } from "ethers"; import { z } from "zod"; diff --git a/packages/sdk/src/schema/contracts/custom.ts b/packages/sdk/src/schema/contracts/custom.ts index 96b144e5399..6824ecc82b3 100644 --- a/packages/sdk/src/schema/contracts/custom.ts +++ b/packages/sdk/src/schema/contracts/custom.ts @@ -1,5 +1,5 @@ import { toSemver } from "../../common/index"; -import { AddressSchema, BigNumberishSchema, JsonSchema } from "../shared"; +import { AddressSchema, BigNumberishSchema } from "../shared"; import { CommonContractOutputSchema, CommonContractSchema, @@ -10,7 +10,7 @@ import { CommonTrustedForwarderSchema, MerkleSchema, } from "./common"; -import { FileOrBufferOrStringSchema } from "@thirdweb-dev/storage"; +import { FileOrBufferOrStringSchema, JsonSchema } from "@thirdweb-dev/storage"; import { BigNumberish } from "ethers"; import { z } from "zod"; diff --git a/packages/sdk/src/schema/shared.ts b/packages/sdk/src/schema/shared.ts index 49fa31971ef..ad81afad6c5 100644 --- a/packages/sdk/src/schema/shared.ts +++ b/packages/sdk/src/schema/shared.ts @@ -1,4 +1,4 @@ -import { Json } from "../core/types"; +import { Json } from "@thirdweb-dev/storage"; import { BigNumber, CallOverrides, utils } from "ethers"; import { z } from "zod"; @@ -31,17 +31,6 @@ export const PercentSchema = z .max(100, "Cannot exeed 100%") .min(0, "Cannot be below 0%"); -export const JsonLiteral = z.union([ - z.string(), - z.number(), - z.boolean(), - z.null(), -]); - -export const JsonSchema: z.ZodSchema = z.lazy(() => - z.union([JsonLiteral, z.array(JsonSchema), z.record(JsonSchema)]), -); -export const JsonObjectSchema = z.record(JsonSchema); export const HexColor = z.union([ z .string() diff --git a/packages/sdk/src/schema/tokens/common/index.ts b/packages/sdk/src/schema/tokens/common/index.ts index 85f0a59c4e2..c7c10b5da45 100644 --- a/packages/sdk/src/schema/tokens/common/index.ts +++ b/packages/sdk/src/schema/tokens/common/index.ts @@ -1,6 +1,6 @@ -import { BigNumberSchema, HexColor, JsonSchema } from "../../shared"; +import { BigNumberSchema, HexColor } from "../../shared"; import { OptionalPropertiesInput } from "./properties"; -import { FileOrBufferOrStringSchema } from "@thirdweb-dev/storage"; +import { FileOrBufferOrStringSchema, JsonSchema } from "@thirdweb-dev/storage"; import { z } from "zod"; /** diff --git a/packages/sdk/src/schema/tokens/common/properties.ts b/packages/sdk/src/schema/tokens/common/properties.ts index 49d0e7fc64a..7d001364c78 100644 --- a/packages/sdk/src/schema/tokens/common/properties.ts +++ b/packages/sdk/src/schema/tokens/common/properties.ts @@ -1,4 +1,4 @@ -import { JsonObjectSchema } from "../../shared"; +import { JsonObjectSchema } from "@thirdweb-dev/storage"; import { z } from "zod"; /** diff --git a/packages/solana/src/types/common.ts b/packages/solana/src/types/common.ts index ef10d979d7b..3e59df76541 100644 --- a/packages/solana/src/types/common.ts +++ b/packages/solana/src/types/common.ts @@ -1,4 +1,5 @@ import { Signer, WalletAdapter } from "@metaplex-foundation/js"; +import { JsonObjectSchema } from "@thirdweb-dev/storage"; import { z } from "zod"; export const MAX_BPS = 10_000; @@ -19,10 +20,6 @@ export const JsonLiteral = z.union([ z.null(), ]); -export const JsonSchema: z.ZodSchema = z.lazy(() => - z.union([JsonLiteral, z.array(JsonSchema), z.record(JsonSchema)]), -); -export const JsonObjectSchema = z.record(JsonSchema); export const HexColor = z.union([ z .string() @@ -38,16 +35,6 @@ export const OptionalPropertiesInput = z .union([z.array(JsonObjectSchema), JsonObjectSchema]) .optional(); -type JsonLiteral = boolean | null | number | string; -type JsonLiteralOrFileOrBuffer = JsonLiteral | FileOrBuffer; -export type Json = JsonLiteralOrFileOrBuffer | JsonObject | Json[]; -export type JsonObject = { [key: string]: Json }; -export type BufferOrStringWithName = { - data: Buffer | string; - name?: string; -}; -export type FileOrBuffer = Buffer | BufferOrStringWithName; - export const AmountSchema = z .union([ z.string().regex(/^([0-9]+\.?[0-9]*|\.[0-9]+)$/, "Invalid amount"), diff --git a/packages/solana/src/types/contracts/index.ts b/packages/solana/src/types/contracts/index.ts index 67a40ce4494..3ca4677e059 100644 --- a/packages/solana/src/types/contracts/index.ts +++ b/packages/solana/src/types/contracts/index.ts @@ -1,5 +1,5 @@ -import { AmountSchema, JsonSchema } from "../common"; -import { FileOrBufferOrStringSchema } from "@thirdweb-dev/storage"; +import { AmountSchema } from "../common"; +import { FileOrBufferOrStringSchema, JsonSchema } from "@thirdweb-dev/storage"; import { z } from "zod"; /** diff --git a/packages/solana/src/types/nft.ts b/packages/solana/src/types/nft.ts index a21bc519ea5..26afb85aad7 100644 --- a/packages/solana/src/types/nft.ts +++ b/packages/solana/src/types/nft.ts @@ -1,5 +1,5 @@ -import { HexColor, JsonSchema, OptionalPropertiesInput } from "./common"; -import { FileOrBufferOrStringSchema } from "@thirdweb-dev/storage"; +import { HexColor, OptionalPropertiesInput } from "./common"; +import { FileOrBufferOrStringSchema, JsonSchema } from "@thirdweb-dev/storage"; import { z } from "zod"; /** diff --git a/packages/storage/src/types/data.ts b/packages/storage/src/types/data.ts index cb9442e0269..02d15a7f7c6 100644 --- a/packages/storage/src/types/data.ts +++ b/packages/storage/src/types/data.ts @@ -36,7 +36,7 @@ export type BufferOrStringWithName = { export type FileOrBufferOrString = FileOrBuffer | string; -const JsonSchema: z.ZodType = z.lazy(() => +export const JsonSchema: z.ZodType = z.lazy(() => z.union([ JsonLiteralSchema, JsonObjectSchema, From 054c32f9944e93911e5b215e9baf862af2a3b753 Mon Sep 17 00:00:00 2001 From: adam-maj Date: Sun, 18 Sep 2022 18:59:29 -0700 Subject: [PATCH 19/19] Update mock storage --- packages/sdk/test/mock/MockStorage.ts | 75 ++----------------- packages/solana/test/mock/MockStorage.ts | 75 ++----------------- packages/storage/package.json | 3 +- .../storage/src/core/downloaders/index.ts | 1 + .../src/core/downloaders/mock-downloader.ts | 31 ++++++++ packages/storage/src/core/uploaders/index.ts | 1 + .../src/core/uploaders/mock-uploader.ts | 51 +++++++++++++ packages/storage/src/types/download.ts | 2 + yarn.lock | 5 ++ 9 files changed, 107 insertions(+), 137 deletions(-) create mode 100644 packages/storage/src/core/downloaders/mock-downloader.ts create mode 100644 packages/storage/src/core/uploaders/mock-uploader.ts diff --git a/packages/sdk/test/mock/MockStorage.ts b/packages/sdk/test/mock/MockStorage.ts index d4ef6bd5fe2..03d2c1144ff 100644 --- a/packages/sdk/test/mock/MockStorage.ts +++ b/packages/sdk/test/mock/MockStorage.ts @@ -1,75 +1,14 @@ import { - DEFAULT_GATEWAY_URLS, - FileOrBufferOrString, - GatewayUrls, - IpfsUploadBatchOptions, - isBufferInstance, - isFileInstance, - IStorageDownloader, - IStorageUploader, + MockUploader, + MockDownloader, ThirdwebStorage, } from "@thirdweb-dev/storage"; -import { v4 as uuidv4 } from "uuid"; - -// Store mapping of URIs to files/objects -const ipfs: Record = {}; - -class MockStorageUploader implements IStorageUploader { - async uploadBatch( - data: FileOrBufferOrString[], - options?: IpfsUploadBatchOptions | undefined, - ): Promise { - const cid = uuidv4(); - const uris: string[] = []; - ipfs[cid] = {}; - - let index = options?.rewriteFileNames?.fileStartNumber || 0; - for (const file of data) { - let contents: string; - if (isFileInstance(file)) { - contents = await file.text(); - } else if (isBufferInstance(file)) { - contents = file.toString(); - } else if (typeof file === "string") { - contents = file; - } else { - contents = isBufferInstance(file.data) - ? file.data.toString() - : file.data; - const name = file.name ? file.name : `file_${index}`; - ipfs[cid][name] = contents; - uris.push(`mock://${cid}/${name}`); - continue; - } - - ipfs[cid][index.toString()] = contents; - uris.push(`mock://${cid}/${index}`); - index += 1; - } - - return uris; - } -} - -class MockStorageDownloader implements IStorageDownloader { - gatewayUrls: GatewayUrls = DEFAULT_GATEWAY_URLS; - - async download(url: string): Promise { - const [cid, name] = url.replace("mock://", "").split("/"); - const data = ipfs[cid][name]; - return { - async json() { - return Promise.resolve(JSON.parse(data)); - }, - async text() { - return Promise.resolve(data); - }, - } as Response; - } -} export function MockStorage(): ThirdwebStorage { - const uploader = new MockStorageUploader(); - const downloader = new MockStorageDownloader(); + // Store mapping of URIs to files/objects + const storage = {}; + + const uploader = new MockUploader(storage); + const downloader = new MockDownloader(storage); return new ThirdwebStorage(uploader, downloader); } diff --git a/packages/solana/test/mock/MockStorage.ts b/packages/solana/test/mock/MockStorage.ts index d4ef6bd5fe2..03d2c1144ff 100644 --- a/packages/solana/test/mock/MockStorage.ts +++ b/packages/solana/test/mock/MockStorage.ts @@ -1,75 +1,14 @@ import { - DEFAULT_GATEWAY_URLS, - FileOrBufferOrString, - GatewayUrls, - IpfsUploadBatchOptions, - isBufferInstance, - isFileInstance, - IStorageDownloader, - IStorageUploader, + MockUploader, + MockDownloader, ThirdwebStorage, } from "@thirdweb-dev/storage"; -import { v4 as uuidv4 } from "uuid"; - -// Store mapping of URIs to files/objects -const ipfs: Record = {}; - -class MockStorageUploader implements IStorageUploader { - async uploadBatch( - data: FileOrBufferOrString[], - options?: IpfsUploadBatchOptions | undefined, - ): Promise { - const cid = uuidv4(); - const uris: string[] = []; - ipfs[cid] = {}; - - let index = options?.rewriteFileNames?.fileStartNumber || 0; - for (const file of data) { - let contents: string; - if (isFileInstance(file)) { - contents = await file.text(); - } else if (isBufferInstance(file)) { - contents = file.toString(); - } else if (typeof file === "string") { - contents = file; - } else { - contents = isBufferInstance(file.data) - ? file.data.toString() - : file.data; - const name = file.name ? file.name : `file_${index}`; - ipfs[cid][name] = contents; - uris.push(`mock://${cid}/${name}`); - continue; - } - - ipfs[cid][index.toString()] = contents; - uris.push(`mock://${cid}/${index}`); - index += 1; - } - - return uris; - } -} - -class MockStorageDownloader implements IStorageDownloader { - gatewayUrls: GatewayUrls = DEFAULT_GATEWAY_URLS; - - async download(url: string): Promise { - const [cid, name] = url.replace("mock://", "").split("/"); - const data = ipfs[cid][name]; - return { - async json() { - return Promise.resolve(JSON.parse(data)); - }, - async text() { - return Promise.resolve(data); - }, - } as Response; - } -} export function MockStorage(): ThirdwebStorage { - const uploader = new MockStorageUploader(); - const downloader = new MockStorageDownloader(); + // Store mapping of URIs to files/objects + const storage = {}; + + const uploader = new MockUploader(storage); + const downloader = new MockDownloader(storage); return new ThirdwebStorage(uploader, downloader); } diff --git a/packages/storage/package.json b/packages/storage/package.json index bf66b5bc524..a816e56fb53 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -36,6 +36,7 @@ "dependencies": { "cross-fetch": "^3.1.5", "form-data": "^4.0.0", + "uuid": "^9.0.0", "zod": "^3.11.6" } -} \ No newline at end of file +} diff --git a/packages/storage/src/core/downloaders/index.ts b/packages/storage/src/core/downloaders/index.ts index d61849d80f8..2c2d8104a02 100644 --- a/packages/storage/src/core/downloaders/index.ts +++ b/packages/storage/src/core/downloaders/index.ts @@ -1 +1,2 @@ export { StorageDownloader } from "./storage-downloader"; +export { MockDownloader } from "./mock-downloader"; diff --git a/packages/storage/src/core/downloaders/mock-downloader.ts b/packages/storage/src/core/downloaders/mock-downloader.ts new file mode 100644 index 00000000000..0715611a5e2 --- /dev/null +++ b/packages/storage/src/core/downloaders/mock-downloader.ts @@ -0,0 +1,31 @@ +import { DEFAULT_GATEWAY_URLS } from "../../common/urls"; +import { + GatewayUrls, + IStorageDownloader, + MemoryStorage, +} from "../../types/download"; + +/** + * @internal + */ +export class MockDownloader implements IStorageDownloader { + gatewayUrls: GatewayUrls = DEFAULT_GATEWAY_URLS; + storage: MemoryStorage; + + constructor(storage: MemoryStorage) { + this.storage = storage; + } + + async download(url: string): Promise { + const [cid, name] = url.replace("mock://", "").split("/"); + const data = this.storage[cid][name]; + return { + async json() { + return Promise.resolve(JSON.parse(data)); + }, + async text() { + return Promise.resolve(data); + }, + } as Response; + } +} diff --git a/packages/storage/src/core/uploaders/index.ts b/packages/storage/src/core/uploaders/index.ts index 61c17cd46cb..2b5e2246f37 100644 --- a/packages/storage/src/core/uploaders/index.ts +++ b/packages/storage/src/core/uploaders/index.ts @@ -1 +1,2 @@ export { IpfsUploader } from "./ipfs-uploader"; +export { MockUploader } from "./mock-uploader"; diff --git a/packages/storage/src/core/uploaders/mock-uploader.ts b/packages/storage/src/core/uploaders/mock-uploader.ts new file mode 100644 index 00000000000..aa06e89d1f5 --- /dev/null +++ b/packages/storage/src/core/uploaders/mock-uploader.ts @@ -0,0 +1,51 @@ +import { isBufferInstance, isFileInstance } from "../../common/utils"; +import { FileOrBufferOrString } from "../../types/data"; +import { MemoryStorage } from "../../types/download"; +import { IpfsUploadBatchOptions, IStorageUploader } from "../../types/upload"; +import { v4 as uuidv4 } from "uuid"; + +/** + * @internal + */ +export class MockUploader implements IStorageUploader { + storage: MemoryStorage; + + constructor(storage: MemoryStorage) { + this.storage = storage; + } + + async uploadBatch( + data: FileOrBufferOrString[], + options?: IpfsUploadBatchOptions | undefined, + ): Promise { + const cid = uuidv4(); + const uris: string[] = []; + this.storage[cid] = {}; + + let index = options?.rewriteFileNames?.fileStartNumber || 0; + for (const file of data) { + let contents: string; + if (isFileInstance(file)) { + contents = await file.text(); + } else if (isBufferInstance(file)) { + contents = file.toString(); + } else if (typeof file === "string") { + contents = file; + } else { + contents = isBufferInstance(file.data) + ? file.data.toString() + : file.data; + const name = file.name ? file.name : `file_${index}`; + this.storage[cid][name] = contents; + uris.push(`mock://${cid}/${name}`); + continue; + } + + this.storage[cid][index.toString()] = contents; + uris.push(`mock://${cid}/${index}`); + index += 1; + } + + return uris; + } +} diff --git a/packages/storage/src/types/download.ts b/packages/storage/src/types/download.ts index 78d3bb25fc1..d09c54c8191 100644 --- a/packages/storage/src/types/download.ts +++ b/packages/storage/src/types/download.ts @@ -6,3 +6,5 @@ export interface IStorageDownloader { export type GatewayUrls = { [key: string]: string[]; }; + +export type MemoryStorage = Record>; diff --git a/yarn.lock b/yarn.lock index c1b614e3c3e..d036212a5ae 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14333,6 +14333,11 @@ uuid@^8.3.2: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== +uuid@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5" + integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg== + v8-compile-cache-lib@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf"