From cd8cf3092dd7e05341069278344ba0b95232db21 Mon Sep 17 00:00:00 2001 From: Lautaro Petaccio Date: Sat, 5 Feb 2022 20:51:09 -0300 Subject: [PATCH 1/5] feat: Add support for array buffers --- src/client/BuilderClient.ts | 9 ++++- src/content/content.spec.ts | 46 ++++++++++++++++----- src/content/content.ts | 6 ++- src/content/types.ts | 2 +- src/files/files.spec.ts | 80 +++++++++++++++++++++++-------------- src/files/files.ts | 18 ++++++--- 6 files changed, 109 insertions(+), 52 deletions(-) diff --git a/src/client/BuilderClient.ts b/src/client/BuilderClient.ts index 63844cc..e141c5f 100644 --- a/src/client/BuilderClient.ts +++ b/src/client/BuilderClient.ts @@ -1,8 +1,8 @@ import FormData from 'form-data' import crossFetch from 'cross-fetch' import { Authenticator, AuthIdentity } from 'dcl-crypto' -import { Content } from '../content/types' import { RemoteItem, LocalItem } from '../item/types' +import { Content } from '../content/types' import { ClientError } from './BuilderClient.errors' import { ServerResponse } from './types' @@ -99,6 +99,13 @@ export class BuilderClient { if (Object.keys(newContent).length > 0) { const formData = new FormData() for (const path in newContent) { + if (newContent[path] instanceof Uint8Array) { + formData.append( + item.contents[path], + (newContent[path] as Uint8Array).buffer + ) + } + formData.append(item.contents[path], newContent[path]) } diff --git a/src/content/content.spec.ts b/src/content/content.spec.ts index 0c7ce0a..b80d007 100644 --- a/src/content/content.spec.ts +++ b/src/content/content.spec.ts @@ -1,25 +1,49 @@ +import { TextEncoder } from 'util' import { THUMBNAIL_PATH } from '../item/constants' import { BodyShapeType } from '../item/types' import { computeHashes, prefixContentName, sortContent } from './content' import { RawContent } from './types' describe('when computing the hashes of raw content', () => { - let content: RawContent + let content: RawContent let hashes: Record + let expectedHashes: Record + let encoder: TextEncoder - beforeEach(async () => { - content = { - someThing: Buffer.from('aThing'), - someOtherThing: Buffer.from('someOtherThing') - } - hashes = await computeHashes(content) - }) - - it('should compute the hash of each Uint8Array', () => { - expect(hashes).toEqual({ + beforeEach(() => { + encoder = new TextEncoder() + expectedHashes = { someOtherThing: 'bafkreiatcrqxiuonjtnt2pml2kmfpct6veomwajfiz5x3t2icj22rpt5ly', someThing: 'bafkreig6f2q62jrqswtg4to2yr454aa2amhuevgsyl6oj3qrwshlazufcu' + } + }) + + describe('when the content is in the Uint8Array format', () => { + beforeEach(async () => { + content = { + someThing: encoder.encode('aThing'), + someOtherThing: encoder.encode('someOtherThing') + } + hashes = await computeHashes(content) + }) + + it('should compute the hash of each Uint8Array', () => { + expect(hashes).toEqual(expectedHashes) + }) + }) + + describe('when the content is in the ArrayBuffer format', () => { + beforeEach(async () => { + content = { + someThing: encoder.encode('aThing').buffer, + someOtherThing: encoder.encode('someOtherThing').buffer + } + hashes = await computeHashes(content) + }) + + it('should compute the hash of each ArrayBuffer', () => { + expect(hashes).toEqual(expectedHashes) }) }) }) diff --git a/src/content/content.ts b/src/content/content.ts index 32775d9..4e6f28f 100644 --- a/src/content/content.ts +++ b/src/content/content.ts @@ -32,7 +32,7 @@ export async function computeHashes( */ async function makeContentFile( path: string, - content: string | Uint8Array | Blob + content: string | Uint8Array | Blob | ArrayBuffer ): Promise<{ name: string; content: Uint8Array }> { if (typeof content === 'string') { // This must be polyfilled in the browser @@ -44,6 +44,8 @@ async function makeContentFile( return { name: path, content: new Uint8Array(buffer) } } else if (content instanceof Uint8Array) { return { name: path, content } + } else if (content instanceof ArrayBuffer) { + return { name: path, content: new Uint8Array(content) } } throw new Error( 'Unable to create ContentFile: content must be a string, a Blob or a Uint8Array' @@ -69,7 +71,7 @@ export function prefixContentName( * @param bodyShape - The body shaped used to sort the content. * @param contents - The contents to be sorted. */ -export function sortContent( +export function sortContent( bodyShape: BodyShapeType, contents: RawContent ): SortedContent { diff --git a/src/content/types.ts b/src/content/types.ts index a8f07c3..498538d 100644 --- a/src/content/types.ts +++ b/src/content/types.ts @@ -1,4 +1,4 @@ -export type Content = Uint8Array | Blob +export type Content = Uint8Array | Blob | ArrayBuffer export type SortedContent = { male: Record diff --git a/src/files/files.spec.ts b/src/files/files.spec.ts index 4d5e413..c89fe3b 100644 --- a/src/files/files.spec.ts +++ b/src/files/files.spec.ts @@ -52,7 +52,7 @@ describe('when loading an item file', () => { describe('and the file is a zip model file', () => { let zipFile: JSZip - let zipFileContent: Uint8Array + let zipFileContent: Uint8Array | ArrayBuffer let thumbnailContent: Uint8Array beforeEach(() => { @@ -72,9 +72,9 @@ describe('when loading an item file', () => { }) it('should throw an error signaling that the asset file is invalid', () => { - return expect( - loadFile(fileName, zipFileContent) - ).rejects.toThrow(new InvalidAssetFileError()) + return expect(loadFile(fileName, zipFileContent)).rejects.toThrow( + new InvalidAssetFileError() + ) }) }) @@ -103,9 +103,7 @@ describe('when loading an item file', () => { }) it("should throw an error signaling that the main file isn't included in the representation contents", () => { - return expect( - loadFile(fileName, zipFileContent) - ).rejects.toThrow( + return expect(loadFile(fileName, zipFileContent)).rejects.toThrow( new ModelInRepresentationNotFoundError('some-unkown-file.glb') ) }) @@ -135,9 +133,9 @@ describe('when loading an item file', () => { }) it('should throw an error signaling that the file in the representation contents was not found', () => { - return expect( - loadFile(fileName, zipFileContent) - ).rejects.toThrow(new FileNotFoundError('some-unkown-file.glb')) + return expect(loadFile(fileName, zipFileContent)).rejects.toThrow( + new FileNotFoundError('some-unkown-file.glb') + ) }) }) @@ -173,21 +171,43 @@ describe('when loading an item file', () => { zipFile.file(ASSET_MANIFEST, JSON.stringify(assetFileContent)) zipFile.file(modelFile, modelFileContent) zipFile.file(textureFile, textureFileContent) - zipFileContent = await zipFile.generateAsync({ - type: 'uint8array' + }) + + describe('and the zip file is in the Uint8Array format', () => { + beforeEach(async () => { + zipFileContent = await zipFile.generateAsync({ + type: 'uint8array' + }) + }) + + it('should build the LoadedFile with the zipped contents and the asset file with the same buffer format as the zip', () => { + return expect(loadFile(fileName, zipFileContent)).resolves.toEqual({ + content: { + [modelFile]: modelFileContent, + [textureFile]: textureFileContent, + [THUMBNAIL_PATH]: thumbnailContent + }, + asset: assetFileContent + }) }) }) - it('should build the LoadedFile with the zipped contents and the asset file', () => { - return expect( - loadFile(fileName, zipFileContent) - ).resolves.toEqual({ - content: { - [modelFile]: modelFileContent, - [textureFile]: textureFileContent, - [THUMBNAIL_PATH]: thumbnailContent - }, - asset: assetFileContent + describe('and the zip file is in the ArrayBuffer format', () => { + beforeEach(async () => { + zipFileContent = await zipFile.generateAsync({ + type: 'arraybuffer' + }) + }) + + it('should build the LoadedFile with the zipped contents and the asset file with the same buffer format as the zip', () => { + return expect(loadFile(fileName, zipFileContent)).resolves.toEqual({ + content: { + [modelFile]: modelFileContent.buffer, + [textureFile]: textureFileContent.buffer, + [THUMBNAIL_PATH]: thumbnailContent.buffer + }, + asset: assetFileContent + }) }) }) }) @@ -204,9 +224,9 @@ describe('when loading an item file', () => { }) it('should throw an error signaling that the file is too big', () => { - return expect( - loadFile(fileName, zipFileContent) - ).rejects.toThrow(new FileTooBigError(modelFile, MAX_FILE_SIZE + 1)) + return expect(loadFile(fileName, zipFileContent)).rejects.toThrow( + new FileTooBigError(modelFile, MAX_FILE_SIZE + 1) + ) }) }) @@ -223,9 +243,9 @@ describe('when loading an item file', () => { }) it("should throw an error signaling that the zip doesn't contain a model file", () => { - return expect( - loadFile(fileName, zipFileContent) - ).rejects.toThrow(new ModelFileNotFoundError()) + return expect(loadFile(fileName, zipFileContent)).rejects.toThrow( + new ModelFileNotFoundError() + ) }) }) @@ -237,9 +257,7 @@ describe('when loading an item file', () => { }) it('should build the LoadedFile contents with the zipped files and the main model file path', () => { - return expect( - loadFile(fileName, zipFileContent) - ).resolves.toEqual({ + return expect(loadFile(fileName, zipFileContent)).resolves.toEqual({ content: { [mainModel]: modelContent, [THUMBNAIL_PATH]: thumbnailContent diff --git a/src/files/files.ts b/src/files/files.ts index d752b7d..008418c 100644 --- a/src/files/files.ts +++ b/src/files/files.ts @@ -1,4 +1,4 @@ -import JSZip from 'jszip' +import JSZip, { OutputType } from 'jszip' import { basename } from 'path' import Ajv from 'ajv' import addAjvFormats from 'ajv-formats' @@ -69,23 +69,29 @@ function isModelPath(fileName: string) { * @param zipFile - The ZIP file. */ async function handleZippedModelFiles( - zipFile: T, - asBlob = false + zipFile: T ): Promise> { const zip: JSZip = await JSZip.loadAsync(zipFile) const fileNames: string[] = [] const promiseOfFileContents: Array> = [] + let fileFormat: OutputType + if (globalThis.Blob && zipFile instanceof globalThis.Blob) { + fileFormat = 'blob' + } else if (zipFile instanceof Uint8Array) { + fileFormat = 'uint8array' + } else { + fileFormat = 'arraybuffer' + } + zip.forEach((filePath, file) => { if ( !basename(filePath).startsWith('.') && basename(filePath) !== ASSET_MANIFEST ) { fileNames.push(filePath) - promiseOfFileContents.push( - file.async(asBlob ? 'blob' : 'uint8array') as Promise - ) + promiseOfFileContents.push(file.async(fileFormat) as Promise) } }) From 9a257846a0658c4ddac4d9856d97daba8200117e Mon Sep 17 00:00:00 2001 From: Lautaro Petaccio Date: Sun, 6 Feb 2022 12:53:26 -0300 Subject: [PATCH 2/5] feat: Add Buffer support --- src/client/BuilderClient.ts | 37 +++++++++++++++++++++++++++++-------- src/content/content.spec.ts | 14 ++++++++++++++ src/content/content.ts | 5 ++++- src/content/types.ts | 2 +- src/files/files.spec.ts | 19 +++++++++++++++++++ src/files/files.ts | 5 +++-- 6 files changed, 70 insertions(+), 12 deletions(-) diff --git a/src/client/BuilderClient.ts b/src/client/BuilderClient.ts index e141c5f..6244f1d 100644 --- a/src/client/BuilderClient.ts +++ b/src/client/BuilderClient.ts @@ -1,4 +1,5 @@ import FormData from 'form-data' +import { Buffer } from 'buffer' import crossFetch from 'cross-fetch' import { Authenticator, AuthIdentity } from 'dcl-crypto' import { RemoteItem, LocalItem } from '../item/types' @@ -56,6 +57,30 @@ export class BuilderClient { return headers } + private convertToFormDataBuffer(data: Content): Blob | Buffer { + const blobExists = globalThis.Blob !== undefined + const bufferExists = Buffer !== undefined + if ( + (blobExists && data instanceof globalThis.Blob) || + (bufferExists && Buffer.isBuffer(data)) + ) { + return data + } + + if ( + blobExists && + (data instanceof Uint8Array || data instanceof ArrayBuffer) + ) { + return new Blob([data]) + } else if (bufferExists && data instanceof Uint8Array) { + return Buffer.from(data.buffer) + } else if (bufferExists && data instanceof ArrayBuffer) { + return Buffer.from(data) + } + + throw new Error('Unsupported content type') + } + /** * Updates or inserts an item. * @param item - The item to insert or update. @@ -99,14 +124,10 @@ export class BuilderClient { if (Object.keys(newContent).length > 0) { const formData = new FormData() for (const path in newContent) { - if (newContent[path] instanceof Uint8Array) { - formData.append( - item.contents[path], - (newContent[path] as Uint8Array).buffer - ) - } - - formData.append(item.contents[path], newContent[path]) + formData.append( + item.contents[path], + this.convertToFormDataBuffer(newContent[path]) + ) } let uploadResponse: Response diff --git a/src/content/content.spec.ts b/src/content/content.spec.ts index b80d007..c60462f 100644 --- a/src/content/content.spec.ts +++ b/src/content/content.spec.ts @@ -46,6 +46,20 @@ describe('when computing the hashes of raw content', () => { expect(hashes).toEqual(expectedHashes) }) }) + + describe('when the content is in the Buffer format', () => { + beforeEach(async () => { + content = { + someThing: Buffer.from('aThing'), + someOtherThing: Buffer.from('someOtherThing') + } + hashes = await computeHashes(content) + }) + + it('should compute the hash of each Buffer', () => { + expect(hashes).toEqual(expectedHashes) + }) + }) }) describe('when prefixing the content name', () => { diff --git a/src/content/content.ts b/src/content/content.ts index 4e6f28f..c786784 100644 --- a/src/content/content.ts +++ b/src/content/content.ts @@ -1,4 +1,5 @@ import { Hashing } from 'dcl-catalyst-commons' +import { Buffer } from 'buffer' import { BodyShapeType } from '../item/types' import { THUMBNAIL_PATH } from '../item/constants' import { Content, HashedContent, RawContent, SortedContent } from './types' @@ -32,7 +33,7 @@ export async function computeHashes( */ async function makeContentFile( path: string, - content: string | Uint8Array | Blob | ArrayBuffer + content: string | Uint8Array | Blob | ArrayBuffer | Buffer ): Promise<{ name: string; content: Uint8Array }> { if (typeof content === 'string') { // This must be polyfilled in the browser @@ -46,6 +47,8 @@ async function makeContentFile( return { name: path, content } } else if (content instanceof ArrayBuffer) { return { name: path, content: new Uint8Array(content) } + } else if (Buffer.isBuffer(content)) { + return { name: path, content: new Uint8Array(content) } } throw new Error( 'Unable to create ContentFile: content must be a string, a Blob or a Uint8Array' diff --git a/src/content/types.ts b/src/content/types.ts index 498538d..f9ad31a 100644 --- a/src/content/types.ts +++ b/src/content/types.ts @@ -1,4 +1,4 @@ -export type Content = Uint8Array | Blob | ArrayBuffer +export type Content = Uint8Array | Blob | ArrayBuffer | Buffer export type SortedContent = { male: Record diff --git a/src/files/files.spec.ts b/src/files/files.spec.ts index c89fe3b..99ba6ff 100644 --- a/src/files/files.spec.ts +++ b/src/files/files.spec.ts @@ -210,6 +210,25 @@ describe('when loading an item file', () => { }) }) }) + + describe('and the zip file is in the Buffer format', () => { + beforeEach(async () => { + zipFileContent = await zipFile.generateAsync({ + type: 'nodebuffer' + }) + }) + + it('should build the LoadedFile with the zipped contents and the asset file with the same buffer format as the zip', () => { + return expect(loadFile(fileName, zipFileContent)).resolves.toEqual({ + content: { + [modelFile]: Buffer.from(modelFileContent.buffer), + [textureFile]: Buffer.from(textureFileContent.buffer), + [THUMBNAIL_PATH]: Buffer.from(thumbnailContent.buffer) + }, + asset: assetFileContent + }) + }) + }) }) }) diff --git a/src/files/files.ts b/src/files/files.ts index 008418c..883c1b5 100644 --- a/src/files/files.ts +++ b/src/files/files.ts @@ -75,13 +75,14 @@ async function handleZippedModelFiles( const fileNames: string[] = [] const promiseOfFileContents: Array> = [] - let fileFormat: OutputType if (globalThis.Blob && zipFile instanceof globalThis.Blob) { fileFormat = 'blob' + } else if (Buffer.isBuffer(zipFile)) { + fileFormat = 'nodebuffer' } else if (zipFile instanceof Uint8Array) { fileFormat = 'uint8array' - } else { + } else if (zipFile instanceof ArrayBuffer) { fileFormat = 'arraybuffer' } From e504125ad4ea89374216faffcbb185630462781d Mon Sep 17 00:00:00 2001 From: Lautaro Petaccio Date: Sun, 6 Feb 2022 13:30:01 -0300 Subject: [PATCH 3/5] chore: Fake commit --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 03f5319..a202d07 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ [![Install Size](https://packagephobia.now.sh/badge?p=@dcl/builder-client@latest)](https://packagephobia.now.sh/result?p=@dcl/builder-client@latest) [![Coverage Status](https://coveralls.io/repos/github/decentraland/builder-client/badge.svg?branch=main)](https://coveralls.io/github/decentraland/builder-client?branch=main) + ## Using the Builder client Using the builder client requires an `AuthIdentity` to be created. From 5aa1250eee01da4b489e09405e6fcc0317b4ca94 Mon Sep 17 00:00:00 2001 From: Lautaro Petaccio Date: Sun, 6 Feb 2022 13:36:16 -0300 Subject: [PATCH 4/5] fix Readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index a202d07..03f5319 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ [![Install Size](https://packagephobia.now.sh/badge?p=@dcl/builder-client@latest)](https://packagephobia.now.sh/result?p=@dcl/builder-client@latest) [![Coverage Status](https://coveralls.io/repos/github/decentraland/builder-client/badge.svg?branch=main)](https://coveralls.io/github/decentraland/builder-client?branch=main) - ## Using the Builder client Using the builder client requires an `AuthIdentity` to be created. From 8bf41dbded66d9b82f95f2b240992a1282830faa Mon Sep 17 00:00:00 2001 From: Lautaro Petaccio Date: Mon, 7 Feb 2022 13:12:36 -0300 Subject: [PATCH 5/5] fix: Rename convertoToFormDataBuffer to convertToFormDataBinary --- src/client/BuilderClient.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/BuilderClient.ts b/src/client/BuilderClient.ts index 6244f1d..5c7ba2d 100644 --- a/src/client/BuilderClient.ts +++ b/src/client/BuilderClient.ts @@ -57,7 +57,7 @@ export class BuilderClient { return headers } - private convertToFormDataBuffer(data: Content): Blob | Buffer { + private convertToFormDataBinary(data: Content): Blob | Buffer { const blobExists = globalThis.Blob !== undefined const bufferExists = Buffer !== undefined if ( @@ -126,7 +126,7 @@ export class BuilderClient { for (const path in newContent) { formData.append( item.contents[path], - this.convertToFormDataBuffer(newContent[path]) + this.convertToFormDataBinary(newContent[path]) ) }