[Storage] - allow uploading the same file multiple times in uploadBatch if it is *the same file*

.changeset/silent-teachers-doubt.md

@@ -0,0 +1,5 @@
+---
+"@thirdweb-dev/storage": patch
+---
+
+allow uploading of the same file with the same file name

packages/storage/src/common/utils.ts

@@ -46,6 +46,45 @@ export function isFileOrBuffer(
   );
 }
 
+/**
+ * @internal
+ */
+export function isFileBufferOrStringEqual(input1: any, input2: any): boolean {
+  if (isFileInstance(input1) && isFileInstance(input2)) {
+    // if both are File types, compare the name, size, and last modified date (best guess that these are the same files)
+    if (
+      input1.name === input2.name &&
+      input1.lastModified === input2.lastModified &&
+      input1.size === input2.size
+    ) {
+      return true;
+    }
+  } else if (isBufferInstance(input1) && isBufferInstance(input2)) {
+    // buffer gives us an easy way to compare the contents!
+
+    return input1.equals(input2);
+  } else if (
+    isBufferOrStringWithName(input1) &&
+    isBufferOrStringWithName(input2)
+  ) {
+    // first check the names
+    if (input1.name === input2.name) {
+      // if the data for both is a string, compare the strings
+      if (typeof input1.data === "string" && typeof input2.data === "string") {
+        return input1.data === input2.data;
+      } else if (
+        isBufferInstance(input1.data) &&
+        isBufferInstance(input2.data)
+      ) {
+        // otherwise we know it's buffers, so compare the buffers
+        return input1.data.equals(input2.data);
+      }
+    }
+  }
+  // otherwise if we have not found a match, return false
+  return false;
+}
+
 /**
  * @internal
  */

packages/storage/src/core/uploaders/ipfs-uploader.ts

@@ -2,6 +2,7 @@ import { PINATA_IPFS_URL, TW_IPFS_SERVER_URL } from "../../common/urls";
 import {
   isBrowser,
   isBufferOrStringWithName,
+  isFileBufferOrStringEqual,
   isFileInstance,
 } from "../../common/utils";
 import {
@@ -85,8 +86,10 @@ export class IpfsUploader implements IStorageUploader<IpfsUploadBatchOptions> {
     files: FileOrBufferOrString[],
     options?: IpfsUploadBatchOptions,
   ) {
+    const fileNameToFileMap = new Map<string, FileOrBufferOrString>();
     const fileNames: string[] = [];
-    files.forEach((file, i) => {
+    for (let i = 0; i < files.length; i++) {
+      const file = files[i];
       let fileName = "";
       let fileData = file;
 
@@ -125,12 +128,23 @@ export class IpfsUploader implements IStorageUploader<IpfsUploadBatchOptions> {
         ? `files`
         : `files/${fileName}`;
 
-      if (fileNames.indexOf(fileName) > -1) {
+      if (fileNameToFileMap.has(fileName)) {
+        // if the file in the map is the same as the file we are already looking at then just skip and continue
+        if (isFileBufferOrStringEqual(fileNameToFileMap.get(fileName), file)) {
+          // we add it to the filenames array so that we can return the correct number of urls,
+          fileNames.push(fileName);
+          // but then we skip because we don't need to upload it multiple times
+          continue;
+        }
+        // otherwise if file names are the same but they are not the same file then we should throw an error (trying to upload to differnt files but with the same names)
         throw new Error(
-          `[DUPLICATE_FILE_NAME_ERROR] File name ${fileName} was passed for more than one file.`,
+          `[DUPLICATE_FILE_NAME_ERROR] File name ${fileName} was passed for more than one different file.`,
         );
       }
 
+      // add it to the map so that we can check for duplicates
+      fileNameToFileMap.set(fileName, file);
+      // add it to the filenames array so that we can return the correct number of urls
       fileNames.push(fileName);
       if (!isBrowser()) {
         form.append("file", fileData as any, { filepath } as any);
@@ -139,7 +153,7 @@ export class IpfsUploader implements IStorageUploader<IpfsUploadBatchOptions> {
         // pls pinata?
         form.append("file", new Blob([fileData as any]), filepath);
       }
-    });
+    }
 
     const metadata = { name: `Storage SDK`, keyvalues: {} };
     form.append("pinataMetadata", JSON.stringify(metadata));

packages/storage/test/ipfs.test.ts

@@ -287,7 +287,7 @@ describe("IPFS", async () => {
     );
   });
 
-  it("Should throw an error when trying to upload files with the same name", async () => {
+  it("Should throw an error when trying to upload different files with the same name", async () => {
     try {
       await storage.uploadBatch([
         {
@@ -302,11 +302,29 @@ describe("IPFS", async () => {
       expect.fail("Uploading files with same name did not throw an error.");
     } catch (err: any) {
       expect(err.message).to.contain(
-        "[DUPLICATE_FILE_NAME_ERROR] File name 0.jpg was passed for more than one file.",
+        "[DUPLICATE_FILE_NAME_ERROR] File name 0.jpg",
       );
     }
   });
 
+  it("Should allow to batch upload the same file multiple times even if they have the same name", async () => {
+    const fileNameWithBufferOne = {
+      name: "0.jpg",
+      data: readFileSync("test/files/0.jpg"),
+    };
+    const fileNameWithBufferTwo = {
+      name: "0.jpg",
+      data: readFileSync("test/files/0.jpg"),
+    };
+
+    const uris = await storage.uploadBatch([
+      fileNameWithBufferOne,
+      fileNameWithBufferTwo,
+    ]);
+
+    expect(uris[0]).to.equal(uris[1]);
+  });
+
   it("Should recursively upload and replace files", async () => {
     // Should test nested within objects and arrays
     const uris = await storage.uploadBatch([