Add resolveScheme method to storage (#156)

Author: adam-maj

.changeset/lucky-penguins-cover.md

@@ -0,0 +1,5 @@
+---
+"@thirdweb-dev/storage": patch
+---
+
+Add resolveScheme method to storage

packages/storage/src/core/downloaders/storage-downloader.ts

@@ -11,38 +11,20 @@ import fetch from "cross-fetch";
  * // Can instantiate the downloader with the default gateway URLs
  * const downloader = new StorageDownloader();
  * const storage = new ThirdwebStorage(undefined, downloader);
- *
- * // Or optionally, can specify your own mapping of URLs
- * const gatewayUrls = {
- *   // We define a mapping of schemes to gateway URLs
- *   "ipfs://": [
- *     "https://gateway.ipfscdn.io/ipfs/",
- *     "https://cloudflare-ipfs.com/ipfs/",
- *     "https://ipfs.io/ipfs/",
- *   ],
- * };
- * const downloader = new StorageDownloader(gatewayUrls);
- * const storage = new ThirdwebStorage(undefined, downloader);
  * ```
  *
  * @public
  */
 export class StorageDownloader implements IStorageDownloader {
-  public gatewayUrls: GatewayUrls;
-
-  constructor(gatewayUrls?: GatewayUrls) {
-    this.gatewayUrls = prepareGatewayUrls(gatewayUrls);
-  }
-
-  async download(uri: string, attempts = 0): Promise<Response> {
+  async download(
+    uri: string,
+    gatewayUrls: GatewayUrls,
+    attempts = 0,
+  ): Promise<Response> {
     // Replace recognized scheme with the highest priority gateway URL that hasn't already been attempted
     let resolvedUri;
     try {
-      resolvedUri = replaceSchemeWithGatewayUrl(
-        uri,
-        this.gatewayUrls,
-        attempts,
-      );
+      resolvedUri = replaceSchemeWithGatewayUrl(uri, gatewayUrls, attempts);
     } catch (err: any) {
       // If every gateway URL we know about for the designated scheme has been tried (via recursion) and failed, throw an error
       if (err.includes("[GATEWAY_URL_ERROR]")) {
@@ -61,7 +43,7 @@ export class StorageDownloader implements IStorageDownloader {
       console.warn(
         `Request to ${resolvedUri} failed with status ${res.status} - ${res.statusText}`,
       );
-      return this.download(uri, attempts + 1);
+      return this.download(uri, gatewayUrls, attempts + 1);
     }
 
     return res;

packages/storage/src/core/storage.ts

@@ -1,12 +1,15 @@
+import { prepareGatewayUrls } from "../common";
 import {
   extractObjectFiles,
   replaceObjectFilesWithUris,
   replaceObjectGatewayUrlsWithSchemes,
   replaceObjectSchemesWithGatewayUrls,
+  replaceSchemeWithGatewayUrl,
 } from "../common/utils";
 import {
   FileOrBuffer,
   FileOrBufferOrStringArraySchema,
+  GatewayUrls,
   IpfsUploadBatchOptions,
   IStorageDownloader,
   IStorageUploader,
@@ -28,24 +31,52 @@ import { IpfsUploader } from "./uploaders/ipfs-uploader";
  * const uri = await storage.upload(data);
  * const result = await storage.download(uri);
  *
- * // Or configure a custom uploader and downloader
+ * // Or configure a custom uploader, downloader, and gateway URLs
+ * const gatewayUrls = {
+ *   // We define a mapping of schemes to gateway URLs
+ *   "ipfs://": [
+ *     "https://gateway.ipfscdn.io/ipfs/",
+ *     "https://cloudflare-ipfs.com/ipfs/",
+ *     "https://ipfs.io/ipfs/",
+ *   ],
+ * };
  * const downloader = new StorageDownloader();
  * const uploader = new IpfsUploader();
- * const storage = new ThirdwebStorage(uploader, downloader)
+ * const storage = new ThirdwebStorage(uploader, downloader, gatewayUrls)
  * ```
  *
  * @public
  */
 export class ThirdwebStorage<T extends UploadOptions = IpfsUploadBatchOptions> {
   private uploader: IStorageUploader<T>;
   private downloader: IStorageDownloader;
+  public gatewayUrls: GatewayUrls;
 
   constructor(
     uploader: IStorageUploader<T> = new IpfsUploader(),
     downloader: IStorageDownloader = new StorageDownloader(),
+    gatewayUrls?: GatewayUrls,
   ) {
     this.uploader = uploader;
     this.downloader = downloader;
+    this.gatewayUrls = prepareGatewayUrls(gatewayUrls);
+  }
+
+  /**
+   * Resolve any scheme on a URL to get a retrievable URL for the data
+   *
+   * @param url - The URL to resolve the scheme of
+   * @returns The URL with its scheme resolved
+   *
+   * @example
+   * ```jsx
+   * const uri = "ipfs://example";
+   * const url = storage.resolveScheme(uri);
+   * console.log(url);
+   * ```
+   */
+  resolveScheme(url: string): string {
+    return replaceSchemeWithGatewayUrl(url, this.gatewayUrls);
   }
 
   /**
@@ -61,7 +92,7 @@ export class ThirdwebStorage<T extends UploadOptions = IpfsUploadBatchOptions> {
    * ```
    */
   async download(url: string): Promise<Response> {
-    return this.downloader.download(url);
+    return this.downloader.download(url, this.gatewayUrls);
   }
 
   /**
@@ -73,7 +104,7 @@ export class ThirdwebStorage<T extends UploadOptions = IpfsUploadBatchOptions> {
    *
    * @example
    * ```jsx
-   * const uri = "ipfs://example"
+   * const uri = "ipfs://example";
    * const json = await storage.downloadJSON(uri);
    * ```
    */
@@ -82,10 +113,7 @@ export class ThirdwebStorage<T extends UploadOptions = IpfsUploadBatchOptions> {
 
     // If we get a JSON object, recursively replace any schemes with gatewayUrls
     const json = await res.json();
-    return replaceObjectSchemesWithGatewayUrls(
-      json,
-      this.downloader.gatewayUrls,
-    ) as TJSON;
+    return replaceObjectSchemesWithGatewayUrls(json, this.gatewayUrls) as TJSON;
   }
 
   /**
@@ -175,19 +203,19 @@ export class ThirdwebStorage<T extends UploadOptions = IpfsUploadBatchOptions> {
   ): Promise<Json[]> {
     let cleaned = data;
     // TODO: Gateway URLs should probably be top-level since both uploader and downloader need them
-    if (this.uploader.gatewayUrls || this.downloader.gatewayUrls) {
+    if (this.gatewayUrls) {
       // Replace any gateway URLs with their hashes
       cleaned = replaceObjectGatewayUrlsWithSchemes(
         cleaned,
-        this.uploader.gatewayUrls || this.downloader.gatewayUrls,
+        this.gatewayUrls,
       ) as Json[];
 
       if (options?.uploadWithGatewayUrl || this.uploader.uploadWithGatewayUrl) {
         // If flag is set, replace all schemes with their preferred gateway URL
         // Ex: used for Solana, where services don't resolve schemes for you, so URLs must be useable by default
         cleaned = replaceObjectSchemesWithGatewayUrls(
           cleaned,
-          this.uploader.gatewayUrls || this.downloader.gatewayUrls,
+          this.gatewayUrls,
         ) as Json[];
       }
     }

packages/storage/src/core/uploaders/ipfs-uploader.ts

@@ -28,17 +28,7 @@ import FormData from "form-data";
  * const storage = new ThirdwebStorage(uploader);
  *
  * // Or optionally, can pass configuration
- * const gatewayUrls = {
- *   // We define a mapping of schemes to gateway URLs
- *   "ipfs://": [
- *     "https://gateway.ipfscdn.io/ipfs/",
- *     "https://cloudflare-ipfs.com/ipfs/",
- *     "https://ipfs.io/ipfs/",
- *   ],
- * };
  * const options = {
- *   // Define cutom gateway URLs
- *   gatewayUrls,
  *   // Upload objects with resolvable URLs
  *   uploadWithGatewayUrl: true,
  * }
@@ -48,11 +38,9 @@ import FormData from "form-data";
  * @public
  */
 export class IpfsUploader implements IStorageUploader<IpfsUploadBatchOptions> {
-  public gatewayUrls: GatewayUrls;
   public uploadWithGatewayUrl: boolean;
 
   constructor(options?: IpfsUploaderOptions) {
-    this.gatewayUrls = prepareGatewayUrls(options?.gatewayUrls);
     this.uploadWithGatewayUrl = options?.uploadWithGatewayUrl || false;
   }
 

packages/storage/src/types/download.ts

@@ -2,17 +2,14 @@
  * @public
  */
 export interface IStorageDownloader {
-  /**
-   * Gateway URLs used to replace schemes on download
-   */
-  gatewayUrls: GatewayUrls;
   /**
    * Download arbitrary data from any URL scheme
    *
    * @param url - The URL to download data from
+   * @param gatewayUrls - The gateway URLs to use for this download
    * @returns The response object of the fetch
    */
-  download(url: string): Promise<Response>;
+  download(url: string, gatewayUrls?: GatewayUrls): Promise<Response>;
 }
 
 /**

packages/storage/src/types/upload.ts

@@ -10,10 +10,6 @@ export type UploadOptions = { [key: string]: any };
  * @public
  */
 export interface IStorageUploader<T extends UploadOptions> {
-  /**
-   * If specified, will be used to replace any gateway URLs with schemes on upload
-   */
-  gatewayUrls?: GatewayUrls;
   /**
    * If specified, will upload objects with gateway URLs instead of schemes
    */
@@ -47,10 +43,6 @@ export type UploadProgressEvent = {
  * @public
  */
 export type IpfsUploaderOptions = {
-  /**
-   * Mapping of URL schemes to gateway URLs to resolve to
-   */
-  gatewayUrls?: GatewayUrls;
   /**
    * Whether or not to replace any URLs with schemes with resolved URLs before upload
    */

packages/storage/test/ipfs.test.ts

@@ -7,6 +7,12 @@ import { readFileSync } from "fs";
 describe("IPFS", async () => {
   const storage = new ThirdwebStorage();
 
+  it("Should resolve scheme with gateway URL", async () => {
+    const uri = `ipfs://example`;
+    const url = storage.resolveScheme(uri);
+    expect(url).to.equal(`${DEFAULT_GATEWAY_URLS["ipfs://"][0]}example`);
+  });
+
   it("Should upload buffer with file number", async () => {
     const uri = await storage.upload(readFileSync("test/files/0.jpg"));